UP
This commit is contained in:
		
							
								
								
									
										170
									
								
								src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										170
									
								
								src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,170 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Cli.Commands; | ||||
| using StellaOps.Cli.Configuration; | ||||
| using StellaOps.Cli.Services; | ||||
| using StellaOps.Cli.Services.Models; | ||||
| using StellaOps.Cli.Telemetry; | ||||
| using StellaOps.Cli.Tests.Testing; | ||||
|  | ||||
| namespace StellaOps.Cli.Tests.Commands; | ||||
|  | ||||
| public sealed class CommandHandlersTests | ||||
| { | ||||
|     [Fact] | ||||
|     public async Task HandleExportJobAsync_SetsExitCodeZeroOnSuccess() | ||||
|     { | ||||
|         var original = Environment.ExitCode; | ||||
|         try | ||||
|         { | ||||
|             var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", "/jobs/export:json/1", null)); | ||||
|             var provider = BuildServiceProvider(backend); | ||||
|  | ||||
|             await CommandHandlers.HandleExportJobAsync(provider, "json", delta: false, verbose: false, CancellationToken.None); | ||||
|  | ||||
|             Assert.Equal(0, Environment.ExitCode); | ||||
|             Assert.Equal("export:json", backend.LastJobKind); | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             Environment.ExitCode = original; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task HandleMergeJobAsync_SetsExitCodeOnFailure() | ||||
|     { | ||||
|         var original = Environment.ExitCode; | ||||
|         try | ||||
|         { | ||||
|             var backend = new StubBackendClient(new JobTriggerResult(false, "Job already running", null, null)); | ||||
|             var provider = BuildServiceProvider(backend); | ||||
|  | ||||
|             await CommandHandlers.HandleMergeJobAsync(provider, verbose: false, CancellationToken.None); | ||||
|  | ||||
|             Assert.Equal(1, Environment.ExitCode); | ||||
|             Assert.Equal("merge:reconcile", backend.LastJobKind); | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             Environment.ExitCode = original; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task HandleScannerRunAsync_AutomaticallyUploadsResults() | ||||
|     { | ||||
|         using var tempDir = new TempDirectory(); | ||||
|         var resultsFile = Path.Combine(tempDir.Path, "results", "scan.json"); | ||||
|         var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", null, null)); | ||||
|         var executor = new StubExecutor(new ScannerExecutionResult(0, resultsFile)); | ||||
|         var options = new StellaOpsCliOptions | ||||
|         { | ||||
|             ResultsDirectory = Path.Combine(tempDir.Path, "results") | ||||
|         }; | ||||
|  | ||||
|         var provider = BuildServiceProvider(backend, executor, new StubInstaller(), options); | ||||
|  | ||||
|         Directory.CreateDirectory(Path.Combine(tempDir.Path, "target")); | ||||
|  | ||||
|         var original = Environment.ExitCode; | ||||
|         try | ||||
|         { | ||||
|             await CommandHandlers.HandleScannerRunAsync( | ||||
|                 provider, | ||||
|                 runner: "docker", | ||||
|                 entry: "scanner-image", | ||||
|                 targetDirectory: Path.Combine(tempDir.Path, "target"), | ||||
|                 arguments: Array.Empty<string>(), | ||||
|                 verbose: false, | ||||
|                 cancellationToken: CancellationToken.None); | ||||
|  | ||||
|             Assert.Equal(0, Environment.ExitCode); | ||||
|             Assert.Equal(resultsFile, backend.LastUploadPath); | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             Environment.ExitCode = original; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static IServiceProvider BuildServiceProvider( | ||||
|         IBackendOperationsClient backend, | ||||
|         IScannerExecutor? executor = null, | ||||
|         IScannerInstaller? installer = null, | ||||
|         StellaOpsCliOptions? options = null) | ||||
|     { | ||||
|         var services = new ServiceCollection(); | ||||
|         services.AddSingleton(backend); | ||||
|         services.AddSingleton<ILoggerFactory>(_ => LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug))); | ||||
|         services.AddSingleton(new VerbosityState()); | ||||
|         services.AddSingleton(options ?? new StellaOpsCliOptions | ||||
|         { | ||||
|             ResultsDirectory = Path.Combine(Path.GetTempPath(), $"stellaops-cli-results-{Guid.NewGuid():N}") | ||||
|         }); | ||||
|         services.AddSingleton<IScannerExecutor>(executor ?? new StubExecutor(new ScannerExecutionResult(0, Path.GetTempFileName()))); | ||||
|         services.AddSingleton<IScannerInstaller>(installer ?? new StubInstaller()); | ||||
|  | ||||
|         return services.BuildServiceProvider(); | ||||
|     } | ||||
|  | ||||
|     private sealed class StubBackendClient : IBackendOperationsClient | ||||
|     { | ||||
|         private readonly JobTriggerResult _result; | ||||
|  | ||||
|         public StubBackendClient(JobTriggerResult result) | ||||
|         { | ||||
|             _result = result; | ||||
|         } | ||||
|  | ||||
|         public string? LastJobKind { get; private set; } | ||||
|         public string? LastUploadPath { get; private set; } | ||||
|  | ||||
|         public Task<ScannerArtifactResult> DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken) | ||||
|             => throw new NotImplementedException(); | ||||
|  | ||||
|         public Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) | ||||
|         { | ||||
|             LastUploadPath = filePath; | ||||
|             return Task.CompletedTask; | ||||
|         } | ||||
|  | ||||
|         public Task<JobTriggerResult> TriggerJobAsync(string jobKind, IDictionary<string, object?> parameters, CancellationToken cancellationToken) | ||||
|         { | ||||
|             LastJobKind = jobKind; | ||||
|             return Task.FromResult(_result); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private sealed class StubExecutor : IScannerExecutor | ||||
|     { | ||||
|         private readonly ScannerExecutionResult _result; | ||||
|  | ||||
|         public StubExecutor(ScannerExecutionResult result) | ||||
|         { | ||||
|             _result = result; | ||||
|         } | ||||
|  | ||||
|         public Task<ScannerExecutionResult> RunAsync(string runner, string entry, string targetDirectory, string resultsDirectory, IReadOnlyList<string> arguments, bool verbose, CancellationToken cancellationToken) | ||||
|         { | ||||
|             Directory.CreateDirectory(Path.GetDirectoryName(_result.ResultsPath)!); | ||||
|             if (!File.Exists(_result.ResultsPath)) | ||||
|             { | ||||
|                 File.WriteAllText(_result.ResultsPath, "{}"); | ||||
|             } | ||||
|  | ||||
|             return Task.FromResult(_result); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private sealed class StubInstaller : IScannerInstaller | ||||
|     { | ||||
|         public Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken) | ||||
|             => Task.CompletedTask; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,79 @@ | ||||
| using System; | ||||
| using System.IO; | ||||
| using System.Text.Json; | ||||
| using StellaOps.Cli.Configuration; | ||||
| using Xunit; | ||||
|  | ||||
| namespace StellaOps.Cli.Tests.Configuration; | ||||
|  | ||||
| public sealed class CliBootstrapperTests : IDisposable | ||||
| { | ||||
|     private readonly string _originalDirectory = Directory.GetCurrentDirectory(); | ||||
|     private readonly string _tempDirectory = Path.Combine(Path.GetTempPath(), $"stellaops-cli-tests-{Guid.NewGuid():N}"); | ||||
|  | ||||
|     public CliBootstrapperTests() | ||||
|     { | ||||
|         Directory.CreateDirectory(_tempDirectory); | ||||
|         Directory.SetCurrentDirectory(_tempDirectory); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public void Build_UsesEnvironmentVariablesWhenPresent() | ||||
|     { | ||||
|         Environment.SetEnvironmentVariable("API_KEY", "env-key"); | ||||
|         Environment.SetEnvironmentVariable("STELLAOPS_BACKEND_URL", "https://env-backend.example"); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var (options, _) = CliBootstrapper.Build(Array.Empty<string>()); | ||||
|  | ||||
|             Assert.Equal("env-key", options.ApiKey); | ||||
|             Assert.Equal("https://env-backend.example", options.BackendUrl); | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             Environment.SetEnvironmentVariable("API_KEY", null); | ||||
|             Environment.SetEnvironmentVariable("STELLAOPS_BACKEND_URL", null); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public void Build_FallsBackToAppSettings() | ||||
|     { | ||||
|         WriteAppSettings(new | ||||
|         { | ||||
|             StellaOps = new | ||||
|             { | ||||
|                 ApiKey = "file-key", | ||||
|                 BackendUrl = "https://file-backend.example" | ||||
|             } | ||||
|         }); | ||||
|  | ||||
|         var (options, _) = CliBootstrapper.Build(Array.Empty<string>()); | ||||
|  | ||||
|         Assert.Equal("file-key", options.ApiKey); | ||||
|         Assert.Equal("https://file-backend.example", options.BackendUrl); | ||||
|     } | ||||
|  | ||||
|     public void Dispose() | ||||
|     { | ||||
|         Directory.SetCurrentDirectory(_originalDirectory); | ||||
|         if (Directory.Exists(_tempDirectory)) | ||||
|         { | ||||
|             try | ||||
|             { | ||||
|                 Directory.Delete(_tempDirectory, recursive: true); | ||||
|             } | ||||
|             catch | ||||
|             { | ||||
|                 // Ignored. | ||||
|             } | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static void WriteAppSettings<T>(T payload) | ||||
|     { | ||||
|         var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions { WriteIndented = true }); | ||||
|         File.WriteAllText("appsettings.json", json); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										235
									
								
								src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										235
									
								
								src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,235 @@ | ||||
| using System; | ||||
| using System.IO; | ||||
| using System.Net; | ||||
| using System.Net.Http; | ||||
| using System.Net.Http.Json; | ||||
| using System.Security.Cryptography; | ||||
| using System.Text; | ||||
| using System.Text.Json; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Cli.Configuration; | ||||
| using StellaOps.Cli.Services; | ||||
| using StellaOps.Cli.Services.Models; | ||||
| using StellaOps.Cli.Services.Models.Transport; | ||||
| using StellaOps.Cli.Tests.Testing; | ||||
|  | ||||
| namespace StellaOps.Cli.Tests.Services; | ||||
|  | ||||
| public sealed class BackendOperationsClientTests | ||||
| { | ||||
|     [Fact] | ||||
|     public async Task DownloadScannerAsync_VerifiesDigestAndWritesMetadata() | ||||
|     { | ||||
|         using var temp = new TempDirectory(); | ||||
|  | ||||
|         var contentBytes = Encoding.UTF8.GetBytes("scanner-blob"); | ||||
|         var digestHex = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant(); | ||||
|  | ||||
|         var handler = new StubHttpMessageHandler((request, _) => | ||||
|         { | ||||
|             var response = new HttpResponseMessage(HttpStatusCode.OK) | ||||
|             { | ||||
|                 Content = new ByteArrayContent(contentBytes), | ||||
|                 RequestMessage = request | ||||
|             }; | ||||
|  | ||||
|             response.Headers.Add("X-StellaOps-Digest", $"sha256:{digestHex}"); | ||||
|             response.Content.Headers.LastModified = DateTimeOffset.UtcNow; | ||||
|             response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream"); | ||||
|             return response; | ||||
|         }); | ||||
|  | ||||
|         var httpClient = new HttpClient(handler) | ||||
|         { | ||||
|             BaseAddress = new Uri("https://feedser.example") | ||||
|         }; | ||||
|  | ||||
|         var options = new StellaOpsCliOptions | ||||
|         { | ||||
|             BackendUrl = "https://feedser.example", | ||||
|             ScannerCacheDirectory = temp.Path, | ||||
|             ScannerDownloadAttempts = 1 | ||||
|         }; | ||||
|  | ||||
|         var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); | ||||
|         var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>()); | ||||
|  | ||||
|         var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); | ||||
|         var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: true, CancellationToken.None); | ||||
|  | ||||
|         Assert.False(result.FromCache); | ||||
|         Assert.True(File.Exists(targetPath)); | ||||
|  | ||||
|         var metadataPath = targetPath + ".metadata.json"; | ||||
|         Assert.True(File.Exists(metadataPath)); | ||||
|  | ||||
|         using var document = JsonDocument.Parse(File.ReadAllText(metadataPath)); | ||||
|         Assert.Equal($"sha256:{digestHex}", document.RootElement.GetProperty("digest").GetString()); | ||||
|         Assert.Equal("stable", document.RootElement.GetProperty("channel").GetString()); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task DownloadScannerAsync_ThrowsOnDigestMismatch() | ||||
|     { | ||||
|         using var temp = new TempDirectory(); | ||||
|  | ||||
|         var contentBytes = Encoding.UTF8.GetBytes("scanner-data"); | ||||
|         var handler = new StubHttpMessageHandler((request, _) => | ||||
|         { | ||||
|             var response = new HttpResponseMessage(HttpStatusCode.OK) | ||||
|             { | ||||
|                 Content = new ByteArrayContent(contentBytes), | ||||
|                 RequestMessage = request | ||||
|             }; | ||||
|             response.Headers.Add("X-StellaOps-Digest", "sha256:deadbeef"); | ||||
|             return response; | ||||
|         }); | ||||
|  | ||||
|         var httpClient = new HttpClient(handler) | ||||
|         { | ||||
|             BaseAddress = new Uri("https://feedser.example") | ||||
|         }; | ||||
|  | ||||
|         var options = new StellaOpsCliOptions | ||||
|         { | ||||
|             BackendUrl = "https://feedser.example", | ||||
|             ScannerCacheDirectory = temp.Path, | ||||
|             ScannerDownloadAttempts = 1 | ||||
|         }; | ||||
|  | ||||
|         var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); | ||||
|         var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>()); | ||||
|  | ||||
|         var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); | ||||
|  | ||||
|         await Assert.ThrowsAsync<InvalidOperationException>(() => client.DownloadScannerAsync("stable", targetPath, overwrite: true, verbose: false, CancellationToken.None)); | ||||
|         Assert.False(File.Exists(targetPath)); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task DownloadScannerAsync_RetriesOnFailure() | ||||
|     { | ||||
|         using var temp = new TempDirectory(); | ||||
|  | ||||
|         var successBytes = Encoding.UTF8.GetBytes("success"); | ||||
|         var digestHex = Convert.ToHexString(SHA256.HashData(successBytes)).ToLowerInvariant(); | ||||
|         var attempts = 0; | ||||
|  | ||||
|         var handler = new StubHttpMessageHandler( | ||||
|             (request, _) => | ||||
|             { | ||||
|                 attempts++; | ||||
|                 return new HttpResponseMessage(HttpStatusCode.InternalServerError) | ||||
|                 { | ||||
|                     RequestMessage = request, | ||||
|                     Content = new StringContent("error") | ||||
|                 }; | ||||
|             }, | ||||
|             (request, _) => | ||||
|             { | ||||
|                 attempts++; | ||||
|                 var response = new HttpResponseMessage(HttpStatusCode.OK) | ||||
|                 { | ||||
|                     RequestMessage = request, | ||||
|                     Content = new ByteArrayContent(successBytes) | ||||
|                 }; | ||||
|                 response.Headers.Add("X-StellaOps-Digest", $"sha256:{digestHex}"); | ||||
|                 return response; | ||||
|             }); | ||||
|  | ||||
|         var httpClient = new HttpClient(handler) | ||||
|         { | ||||
|             BaseAddress = new Uri("https://feedser.example") | ||||
|         }; | ||||
|  | ||||
|         var options = new StellaOpsCliOptions | ||||
|         { | ||||
|             BackendUrl = "https://feedser.example", | ||||
|             ScannerCacheDirectory = temp.Path, | ||||
|             ScannerDownloadAttempts = 3 | ||||
|         }; | ||||
|  | ||||
|         var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); | ||||
|         var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>()); | ||||
|  | ||||
|         var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); | ||||
|         var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: false, CancellationToken.None); | ||||
|  | ||||
|         Assert.Equal(2, attempts); | ||||
|         Assert.False(result.FromCache); | ||||
|         Assert.True(File.Exists(targetPath)); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task TriggerJobAsync_ReturnsAcceptedResult() | ||||
|     { | ||||
|         var handler = new StubHttpMessageHandler((request, _) => | ||||
|         { | ||||
|             var response = new HttpResponseMessage(HttpStatusCode.Accepted) | ||||
|             { | ||||
|                 RequestMessage = request, | ||||
|                 Content = JsonContent.Create(new JobRunResponse | ||||
|                 { | ||||
|                     RunId = Guid.NewGuid(), | ||||
|                     Status = "queued", | ||||
|                     Kind = "export:json", | ||||
|                     Trigger = "cli", | ||||
|                     CreatedAt = DateTimeOffset.UtcNow | ||||
|                 }) | ||||
|             }; | ||||
|             response.Headers.Location = new Uri("/jobs/export:json/runs/123", UriKind.Relative); | ||||
|             return response; | ||||
|         }); | ||||
|  | ||||
|         var httpClient = new HttpClient(handler) | ||||
|         { | ||||
|             BaseAddress = new Uri("https://feedser.example") | ||||
|         }; | ||||
|  | ||||
|         var options = new StellaOpsCliOptions { BackendUrl = "https://feedser.example" }; | ||||
|         var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); | ||||
|         var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>()); | ||||
|  | ||||
|         var result = await client.TriggerJobAsync("export:json", new Dictionary<string, object?>(), CancellationToken.None); | ||||
|  | ||||
|         Assert.True(result.Success); | ||||
|         Assert.Equal("Accepted", result.Message); | ||||
|         Assert.Equal("/jobs/export:json/runs/123", result.Location); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task TriggerJobAsync_ReturnsFailureMessage() | ||||
|     { | ||||
|         var handler = new StubHttpMessageHandler((request, _) => | ||||
|         { | ||||
|             var problem = new | ||||
|             { | ||||
|                 title = "Job already running", | ||||
|                 detail = "export job active" | ||||
|             }; | ||||
|  | ||||
|             var response = new HttpResponseMessage(HttpStatusCode.Conflict) | ||||
|             { | ||||
|                 RequestMessage = request, | ||||
|                 Content = JsonContent.Create(problem) | ||||
|             }; | ||||
|             return response; | ||||
|         }); | ||||
|  | ||||
|         var httpClient = new HttpClient(handler) | ||||
|         { | ||||
|             BaseAddress = new Uri("https://feedser.example") | ||||
|         }; | ||||
|  | ||||
|         var options = new StellaOpsCliOptions { BackendUrl = "https://feedser.example" }; | ||||
|         var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); | ||||
|         var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>()); | ||||
|  | ||||
|         var result = await client.TriggerJobAsync("export:json", new Dictionary<string, object?>(), CancellationToken.None); | ||||
|  | ||||
|         Assert.False(result.Success); | ||||
|         Assert.Contains("Job already running", result.Message); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										28
									
								
								src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										28
									
								
								src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,28 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|  | ||||
|   <PropertyGroup> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|     <IsPackable>false</IsPackable> | ||||
|  | ||||
|     <!-- To enable Microsoft.Testing.Platform, uncomment the following line. --> | ||||
|     <!-- <UseMicrosoftTestingPlatformRunner>true</UseMicrosoftTestingPlatformRunner> --> | ||||
|     <!-- Note: to use Microsoft.Testing.Platform correctly with dotnet test: --> | ||||
|     <!-- 1. You must add dotnet.config specifying the test runner to be Microsoft.Testing.Platform --> | ||||
|     <!-- 2. You must use .NET 10 SDK or later --> | ||||
|     <!-- For more information, see https://aka.ms/dotnet-test/mtp and https://xunit.net/docs/getting-started/v3/microsoft-testing-platform --> | ||||
|     <!-- To enable code coverage with Microsoft.Testing.Platform, add a package reference to Microsoft.Testing.Extensions.CodeCoverage --> | ||||
|     <!-- https://learn.microsoft.comdotnet/core/testing/microsoft-testing-platform-extensions-code-coverage --> | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <Using Include="Xunit" /> | ||||
|   </ItemGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="..\StellaOps.Cli\StellaOps.Cli.csproj" /> | ||||
|     <ProjectReference Include="..\StellaOps.Configuration\StellaOps.Configuration.csproj" /> | ||||
|   </ItemGroup> | ||||
|  | ||||
| </Project> | ||||
							
								
								
									
										55
									
								
								src/StellaOps.Cli.Tests/Testing/TestHelpers.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										55
									
								
								src/StellaOps.Cli.Tests/Testing/TestHelpers.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,55 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using System.Net.Http; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
|  | ||||
| namespace StellaOps.Cli.Tests.Testing; | ||||
|  | ||||
| internal sealed class TempDirectory : IDisposable | ||||
| { | ||||
|     public TempDirectory() | ||||
|     { | ||||
|         Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-cli-tests-{Guid.NewGuid():N}"); | ||||
|         Directory.CreateDirectory(Path); | ||||
|     } | ||||
|  | ||||
|     public string Path { get; } | ||||
|  | ||||
|     public void Dispose() | ||||
|     { | ||||
|         try | ||||
|         { | ||||
|             if (Directory.Exists(Path)) | ||||
|             { | ||||
|                 Directory.Delete(Path, recursive: true); | ||||
|             } | ||||
|         } | ||||
|         catch | ||||
|         { | ||||
|             // ignored | ||||
|         } | ||||
|     } | ||||
| } | ||||
|  | ||||
| internal sealed class StubHttpMessageHandler : HttpMessageHandler | ||||
| { | ||||
|     private readonly Queue<Func<HttpRequestMessage, CancellationToken, HttpResponseMessage>> _responses; | ||||
|  | ||||
|     public StubHttpMessageHandler(params Func<HttpRequestMessage, CancellationToken, HttpResponseMessage>[] handlers) | ||||
|     { | ||||
|         if (handlers is null || handlers.Length == 0) | ||||
|         { | ||||
|             throw new ArgumentException("At least one handler must be provided.", nameof(handlers)); | ||||
|         } | ||||
|  | ||||
|         _responses = new Queue<Func<HttpRequestMessage, CancellationToken, HttpResponseMessage>>(handlers); | ||||
|     } | ||||
|  | ||||
|     protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var factory = _responses.Count > 1 ? _responses.Dequeue() : _responses.Peek(); | ||||
|         return Task.FromResult(factory(request, cancellationToken)); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										10
									
								
								src/StellaOps.Cli.Tests/UnitTest1.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										10
									
								
								src/StellaOps.Cli.Tests/UnitTest1.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,10 @@ | ||||
| namespace StellaOps.Cli.Tests; | ||||
|  | ||||
| public class UnitTest1 | ||||
| { | ||||
|     [Fact] | ||||
|     public void Test1() | ||||
|     { | ||||
|  | ||||
|     } | ||||
| } | ||||
							
								
								
									
										3
									
								
								src/StellaOps.Cli.Tests/xunit.runner.json
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										3
									
								
								src/StellaOps.Cli.Tests/xunit.runner.json
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,3 @@ | ||||
| { | ||||
|     "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json" | ||||
| } | ||||
							
								
								
									
										27
									
								
								src/StellaOps.Cli/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										27
									
								
								src/StellaOps.Cli/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,27 @@ | ||||
| # StellaOps.Cli — Agent Brief | ||||
|  | ||||
| ## Mission | ||||
| - Deliver an offline-capable command-line interface that drives StellaOps back-end operations: scanner distribution, scan execution, result uploads, and Feedser database lifecycle calls (init/resume/export). | ||||
| - Honour StellaOps principles of determinism, observability, and offline-first behaviour while providing a polished operator experience. | ||||
|  | ||||
| ## Role Charter | ||||
| | Role | Mandate | Collaboration | | ||||
| | --- | --- | --- | | ||||
| | **DevEx/CLI** | Own CLI UX, command routing, and configuration model. Ensure commands work with empty/default config and document overrides. | Coordinate with Backend/WebService for API contracts and with Docs for operator workflows. | | ||||
| | **Ops Integrator** | Maintain integration paths for shell/dotnet/docker tooling. Validate that air-gapped runners can bootstrap required binaries. | Work with Feedser/Agent teams to mirror packaging and signing requirements. | | ||||
| | **QA** | Provide command-level fixtures, golden outputs, and regression coverage (unit & smoke). Ensure commands respect cancellation and deterministic logging. | Partner with QA guild for shared harnesses and test data. | | ||||
|  | ||||
| ## Working Agreements | ||||
| - Configuration is centralised in `StellaOps.Configuration`; always consume the bootstrapper instead of hand rolling builders. Env vars (`API_KEY`, `STELLAOPS_BACKEND_URL`, `StellaOps:*`) override JSON/YAML and default to empty values. | ||||
| - Command verbs (`scanner`, `scan`, `db`, `config`) are wired through System.CommandLine 2.0; keep handlers composable, cancellation-aware, and unit-testable. | ||||
| - `scanner download` must verify digests/signatures, install containers locally (docker load), and log artefact metadata. | ||||
| - `scan run` must execute the container against a directory, materialise artefacts in `ResultsDirectory`, and auto-upload them on success; `scan upload` is the manual retry path. | ||||
| - Emit structured console logs (single line, UTC timestamps) and honour offline-first expectations—no hidden network calls. | ||||
| - Mirror repository guidance: stay within `src/StellaOps.Cli` unless collaborating via documented handshakes. | ||||
| - Update `TASKS.md` as states change (TODO → DOING → DONE/BLOCKED) and record added tests/fixtures alongside implementation notes. | ||||
|  | ||||
| ## Reference Materials | ||||
| - `docs/ARCHITECTURE_FEEDSER.md` for database operations surface area. | ||||
| - Backend OpenAPI/contract docs (once available) for job triggers and scanner endpoints. | ||||
| - Existing module AGENTS/TASKS files for style and coordination cues. | ||||
| - `docs/09_API_CLI_REFERENCE.md` (section 3) for the user-facing synopsis of the CLI verbs and flags. | ||||
							
								
								
									
										246
									
								
								src/StellaOps.Cli/Commands/CommandFactory.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										246
									
								
								src/StellaOps.Cli/Commands/CommandFactory.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,246 @@ | ||||
| using System; | ||||
| using System.CommandLine; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Cli.Configuration; | ||||
|  | ||||
| namespace StellaOps.Cli.Commands; | ||||
|  | ||||
| internal static class CommandFactory | ||||
| { | ||||
|     public static RootCommand Create(IServiceProvider services, StellaOpsCliOptions options, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var verboseOption = new Option<bool>("--verbose", new[] { "-v" }) | ||||
|         { | ||||
|             Description = "Enable verbose logging output." | ||||
|         }; | ||||
|  | ||||
|         var root = new RootCommand("StellaOps command-line interface") | ||||
|         { | ||||
|             TreatUnmatchedTokensAsErrors = true | ||||
|         }; | ||||
|         root.Add(verboseOption); | ||||
|  | ||||
|         root.Add(BuildScannerCommand(services, verboseOption, cancellationToken)); | ||||
|         root.Add(BuildScanCommand(services, options, verboseOption, cancellationToken)); | ||||
|         root.Add(BuildDatabaseCommand(services, verboseOption, cancellationToken)); | ||||
|         root.Add(BuildConfigCommand(options)); | ||||
|  | ||||
|         return root; | ||||
|     } | ||||
|  | ||||
|     private static Command BuildScannerCommand(IServiceProvider services, Option<bool> verboseOption, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var scanner = new Command("scanner", "Manage scanner artifacts and lifecycle."); | ||||
|  | ||||
|         var download = new Command("download", "Download the latest scanner bundle."); | ||||
|         var channelOption = new Option<string>("--channel", new[] { "-c" }) | ||||
|         { | ||||
|             Description = "Scanner channel (stable, beta, nightly)." | ||||
|         }; | ||||
|  | ||||
|         var outputOption = new Option<string?>("--output") | ||||
|         { | ||||
|             Description = "Optional output path for the downloaded bundle." | ||||
|         }; | ||||
|  | ||||
|         var overwriteOption = new Option<bool>("--overwrite") | ||||
|         { | ||||
|             Description = "Overwrite existing bundle if present." | ||||
|         }; | ||||
|  | ||||
|         var noInstallOption = new Option<bool>("--no-install") | ||||
|         { | ||||
|             Description = "Skip installing the scanner container after download." | ||||
|         }; | ||||
|  | ||||
|         download.Add(channelOption); | ||||
|         download.Add(outputOption); | ||||
|         download.Add(overwriteOption); | ||||
|         download.Add(noInstallOption); | ||||
|  | ||||
|         download.SetAction((parseResult, _) => | ||||
|         { | ||||
|             var channel = parseResult.GetValue(channelOption) ?? "stable"; | ||||
|             var output = parseResult.GetValue(outputOption); | ||||
|             var overwrite = parseResult.GetValue(overwriteOption); | ||||
|             var install = !parseResult.GetValue(noInstallOption); | ||||
|             var verbose = parseResult.GetValue(verboseOption); | ||||
|  | ||||
|             return CommandHandlers.HandleScannerDownloadAsync(services, channel, output, overwrite, install, verbose, cancellationToken); | ||||
|         }); | ||||
|  | ||||
|         scanner.Add(download); | ||||
|         return scanner; | ||||
|     } | ||||
|  | ||||
|     private static Command BuildScanCommand(IServiceProvider services, StellaOpsCliOptions options, Option<bool> verboseOption, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var scan = new Command("scan", "Execute scanners and manage scan outputs."); | ||||
|  | ||||
|         var run = new Command("run", "Execute a scanner bundle with the configured runner."); | ||||
|         var runnerOption = new Option<string>("--runner") | ||||
|         { | ||||
|             Description = "Execution runtime (dotnet, self, docker)." | ||||
|         }; | ||||
|         var entryOption = new Option<string>("--entry") | ||||
|         { | ||||
|             Description = "Path to the scanner entrypoint or Docker image.", | ||||
|             Required = true | ||||
|         }; | ||||
|         var targetOption = new Option<string>("--target") | ||||
|         { | ||||
|             Description = "Directory to scan.", | ||||
|             Required = true | ||||
|         }; | ||||
|  | ||||
|         var argsArgument = new Argument<string[]>("scanner-args") | ||||
|         { | ||||
|             Arity = ArgumentArity.ZeroOrMore | ||||
|         }; | ||||
|  | ||||
|         run.Add(runnerOption); | ||||
|         run.Add(entryOption); | ||||
|         run.Add(targetOption); | ||||
|         run.Add(argsArgument); | ||||
|  | ||||
|         run.SetAction((parseResult, _) => | ||||
|         { | ||||
|             var runner = parseResult.GetValue(runnerOption) ?? options.DefaultRunner; | ||||
|             var entry = parseResult.GetValue(entryOption) ?? string.Empty; | ||||
|             var target = parseResult.GetValue(targetOption) ?? string.Empty; | ||||
|             var forwardedArgs = parseResult.GetValue(argsArgument) ?? Array.Empty<string>(); | ||||
|             var verbose = parseResult.GetValue(verboseOption); | ||||
|  | ||||
|             return CommandHandlers.HandleScannerRunAsync(services, runner, entry, target, forwardedArgs, verbose, cancellationToken); | ||||
|         }); | ||||
|  | ||||
|         var upload = new Command("upload", "Upload completed scan results to the backend."); | ||||
|         var fileOption = new Option<string>("--file") | ||||
|         { | ||||
|             Description = "Path to the scan result artifact.", | ||||
|             Required = true | ||||
|         }; | ||||
|         upload.Add(fileOption); | ||||
|         upload.SetAction((parseResult, _) => | ||||
|         { | ||||
|             var file = parseResult.GetValue(fileOption) ?? string.Empty; | ||||
|             var verbose = parseResult.GetValue(verboseOption); | ||||
|             return CommandHandlers.HandleScanUploadAsync(services, file, verbose, cancellationToken); | ||||
|         }); | ||||
|  | ||||
|         scan.Add(run); | ||||
|         scan.Add(upload); | ||||
|         return scan; | ||||
|     } | ||||
|  | ||||
|     private static Command BuildDatabaseCommand(IServiceProvider services, Option<bool> verboseOption, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var db = new Command("db", "Trigger Feedser database operations via backend jobs."); | ||||
|  | ||||
|         var fetch = new Command("fetch", "Trigger connector fetch/parse/map stages."); | ||||
|         var sourceOption = new Option<string>("--source") | ||||
|         { | ||||
|             Description = "Connector source identifier (e.g. redhat, osv, vmware).", | ||||
|             Required = true | ||||
|         }; | ||||
|         var stageOption = new Option<string>("--stage") | ||||
|         { | ||||
|             Description = "Stage to trigger: fetch, parse, or map." | ||||
|         }; | ||||
|         var modeOption = new Option<string?>("--mode") | ||||
|         { | ||||
|             Description = "Optional connector-specific mode (init, resume, cursor)." | ||||
|         }; | ||||
|  | ||||
|         fetch.Add(sourceOption); | ||||
|         fetch.Add(stageOption); | ||||
|         fetch.Add(modeOption); | ||||
|         fetch.SetAction((parseResult, _) => | ||||
|         { | ||||
|             var source = parseResult.GetValue(sourceOption) ?? string.Empty; | ||||
|             var stage = parseResult.GetValue(stageOption) ?? "fetch"; | ||||
|             var mode = parseResult.GetValue(modeOption); | ||||
|             var verbose = parseResult.GetValue(verboseOption); | ||||
|  | ||||
|             return CommandHandlers.HandleConnectorJobAsync(services, source, stage, mode, verbose, cancellationToken); | ||||
|         }); | ||||
|  | ||||
|         var merge = new Command("merge", "Run canonical merge reconciliation."); | ||||
|         merge.SetAction((parseResult, _) => | ||||
|         { | ||||
|             var verbose = parseResult.GetValue(verboseOption); | ||||
|             return CommandHandlers.HandleMergeJobAsync(services, verbose, cancellationToken); | ||||
|         }); | ||||
|  | ||||
|         var export = new Command("export", "Run Feedser export jobs."); | ||||
|         var formatOption = new Option<string>("--format") | ||||
|         { | ||||
|             Description = "Export format: json or trivy-db." | ||||
|         }; | ||||
|         var deltaOption = new Option<bool>("--delta") | ||||
|         { | ||||
|             Description = "Request a delta export when supported." | ||||
|         }; | ||||
|  | ||||
|         export.Add(formatOption); | ||||
|         export.Add(deltaOption); | ||||
|         export.SetAction((parseResult, _) => | ||||
|         { | ||||
|             var format = parseResult.GetValue(formatOption) ?? "json"; | ||||
|             var delta = parseResult.GetValue(deltaOption); | ||||
|             var verbose = parseResult.GetValue(verboseOption); | ||||
|             return CommandHandlers.HandleExportJobAsync(services, format, delta, verbose, cancellationToken); | ||||
|         }); | ||||
|  | ||||
|         db.Add(fetch); | ||||
|         db.Add(merge); | ||||
|         db.Add(export); | ||||
|         return db; | ||||
|     } | ||||
|  | ||||
|     private static Command BuildConfigCommand(StellaOpsCliOptions options) | ||||
|     { | ||||
|         var config = new Command("config", "Inspect CLI configuration state."); | ||||
|         var show = new Command("show", "Display resolved configuration values."); | ||||
|  | ||||
|         show.SetAction((_, _) => | ||||
|         { | ||||
|             var lines = new[] | ||||
|             { | ||||
|                 $"Backend URL: {MaskIfEmpty(options.BackendUrl)}", | ||||
|                 $"API Key: {DescribeSecret(options.ApiKey)}", | ||||
|                 $"Scanner Cache: {options.ScannerCacheDirectory}", | ||||
|                 $"Results Directory: {options.ResultsDirectory}", | ||||
|                 $"Default Runner: {options.DefaultRunner}" | ||||
|             }; | ||||
|  | ||||
|             foreach (var line in lines) | ||||
|             { | ||||
|                 Console.WriteLine(line); | ||||
|             } | ||||
|  | ||||
|             return Task.CompletedTask; | ||||
|         }); | ||||
|  | ||||
|         config.Add(show); | ||||
|         return config; | ||||
|     } | ||||
|  | ||||
|     private static string MaskIfEmpty(string value) | ||||
|         => string.IsNullOrWhiteSpace(value) ? "<not configured>" : value; | ||||
|  | ||||
|     private static string DescribeSecret(string value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return "<not configured>"; | ||||
|         } | ||||
|  | ||||
|         return value.Length switch | ||||
|         { | ||||
|             <= 4 => "****", | ||||
|             _ => $"{value[..2]}***{value[^2..]}" | ||||
|         }; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										323
									
								
								src/StellaOps.Cli/Commands/CommandHandlers.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										323
									
								
								src/StellaOps.Cli/Commands/CommandHandlers.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,323 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Diagnostics; | ||||
| using System.IO; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Cli.Configuration; | ||||
| using StellaOps.Cli.Services; | ||||
| using StellaOps.Cli.Services.Models; | ||||
| using StellaOps.Cli.Telemetry; | ||||
|  | ||||
| namespace StellaOps.Cli.Commands; | ||||
|  | ||||
| internal static class CommandHandlers | ||||
| { | ||||
|     public static async Task HandleScannerDownloadAsync( | ||||
|         IServiceProvider services, | ||||
|         string channel, | ||||
|         string? output, | ||||
|         bool overwrite, | ||||
|         bool install, | ||||
|         bool verbose, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         await using var scope = services.CreateAsyncScope(); | ||||
|         var client = scope.ServiceProvider.GetRequiredService<IBackendOperationsClient>(); | ||||
|         var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("scanner-download"); | ||||
|         var verbosity = scope.ServiceProvider.GetRequiredService<VerbosityState>(); | ||||
|         var previousLevel = verbosity.MinimumLevel; | ||||
|         verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; | ||||
|         using var activity = CliActivitySource.Instance.StartActivity("cli.scanner.download", ActivityKind.Client); | ||||
|         activity?.SetTag("stellaops.cli.command", "scanner download"); | ||||
|         activity?.SetTag("stellaops.cli.channel", channel); | ||||
|         using var duration = CliMetrics.MeasureCommandDuration("scanner download"); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var result = await client.DownloadScannerAsync(channel, output ?? string.Empty, overwrite, verbose, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             if (result.FromCache) | ||||
|             { | ||||
|                 logger.LogInformation("Using cached scanner at {Path}.", result.Path); | ||||
|             } | ||||
|             else | ||||
|             { | ||||
|                 logger.LogInformation("Scanner downloaded to {Path} ({Size} bytes).", result.Path, result.SizeBytes); | ||||
|             } | ||||
|  | ||||
|             CliMetrics.RecordScannerDownload(channel, result.FromCache); | ||||
|  | ||||
|             if (install) | ||||
|             { | ||||
|                 var installer = scope.ServiceProvider.GetRequiredService<IScannerInstaller>(); | ||||
|                 await installer.InstallAsync(result.Path, verbose, cancellationToken).ConfigureAwait(false); | ||||
|                 CliMetrics.RecordScannerInstall(channel); | ||||
|             } | ||||
|  | ||||
|             Environment.ExitCode = 0; | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogError(ex, "Failed to download scanner bundle."); | ||||
|             Environment.ExitCode = 1; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             verbosity.MinimumLevel = previousLevel; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public static async Task HandleScannerRunAsync( | ||||
|         IServiceProvider services, | ||||
|         string runner, | ||||
|         string entry, | ||||
|         string targetDirectory, | ||||
|         IReadOnlyList<string> arguments, | ||||
|         bool verbose, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         await using var scope = services.CreateAsyncScope(); | ||||
|         var executor = scope.ServiceProvider.GetRequiredService<IScannerExecutor>(); | ||||
|         var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("scanner-run"); | ||||
|         var verbosity = scope.ServiceProvider.GetRequiredService<VerbosityState>(); | ||||
|         var previousLevel = verbosity.MinimumLevel; | ||||
|         verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; | ||||
|         using var activity = CliActivitySource.Instance.StartActivity("cli.scan.run", ActivityKind.Internal); | ||||
|         activity?.SetTag("stellaops.cli.command", "scan run"); | ||||
|         activity?.SetTag("stellaops.cli.runner", runner); | ||||
|         activity?.SetTag("stellaops.cli.entry", entry); | ||||
|         activity?.SetTag("stellaops.cli.target", targetDirectory); | ||||
|         using var duration = CliMetrics.MeasureCommandDuration("scan run"); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var options = scope.ServiceProvider.GetRequiredService<StellaOpsCliOptions>(); | ||||
|             var resultsDirectory = options.ResultsDirectory; | ||||
|  | ||||
|             var executionResult = await executor.RunAsync( | ||||
|                 runner, | ||||
|                 entry, | ||||
|                 targetDirectory, | ||||
|                 resultsDirectory, | ||||
|                 arguments, | ||||
|                 verbose, | ||||
|                 cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             Environment.ExitCode = executionResult.ExitCode; | ||||
|             CliMetrics.RecordScanRun(runner, executionResult.ExitCode); | ||||
|  | ||||
|             if (executionResult.ExitCode == 0) | ||||
|             { | ||||
|                 var backend = scope.ServiceProvider.GetRequiredService<IBackendOperationsClient>(); | ||||
|                 logger.LogInformation("Uploading scan artefact {Path}...", executionResult.ResultsPath); | ||||
|                 await backend.UploadScanResultsAsync(executionResult.ResultsPath, cancellationToken).ConfigureAwait(false); | ||||
|                 logger.LogInformation("Scan artefact uploaded."); | ||||
|                 activity?.SetTag("stellaops.cli.results", executionResult.ResultsPath); | ||||
|             } | ||||
|             else | ||||
|             { | ||||
|                 logger.LogWarning("Skipping automatic upload because scan exited with code {Code}.", executionResult.ExitCode); | ||||
|             } | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogError(ex, "Scanner execution failed."); | ||||
|             Environment.ExitCode = 1; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             verbosity.MinimumLevel = previousLevel; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public static async Task HandleScanUploadAsync( | ||||
|         IServiceProvider services, | ||||
|         string file, | ||||
|         bool verbose, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         await using var scope = services.CreateAsyncScope(); | ||||
|         var client = scope.ServiceProvider.GetRequiredService<IBackendOperationsClient>(); | ||||
|         var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("scanner-upload"); | ||||
|         var verbosity = scope.ServiceProvider.GetRequiredService<VerbosityState>(); | ||||
|         var previousLevel = verbosity.MinimumLevel; | ||||
|         verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; | ||||
|         using var activity = CliActivitySource.Instance.StartActivity("cli.scan.upload", ActivityKind.Client); | ||||
|         activity?.SetTag("stellaops.cli.command", "scan upload"); | ||||
|         activity?.SetTag("stellaops.cli.file", file); | ||||
|         using var duration = CliMetrics.MeasureCommandDuration("scan upload"); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var path = Path.GetFullPath(file); | ||||
|             await client.UploadScanResultsAsync(path, cancellationToken).ConfigureAwait(false); | ||||
|             logger.LogInformation("Scan results uploaded successfully."); | ||||
|             Environment.ExitCode = 0; | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogError(ex, "Failed to upload scan results."); | ||||
|             Environment.ExitCode = 1; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             verbosity.MinimumLevel = previousLevel; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public static async Task HandleConnectorJobAsync( | ||||
|         IServiceProvider services, | ||||
|         string source, | ||||
|         string stage, | ||||
|         string? mode, | ||||
|         bool verbose, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         await using var scope = services.CreateAsyncScope(); | ||||
|         var client = scope.ServiceProvider.GetRequiredService<IBackendOperationsClient>(); | ||||
|         var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("db-connector"); | ||||
|         var verbosity = scope.ServiceProvider.GetRequiredService<VerbosityState>(); | ||||
|         var previousLevel = verbosity.MinimumLevel; | ||||
|         verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; | ||||
|         using var activity = CliActivitySource.Instance.StartActivity("cli.db.fetch", ActivityKind.Client); | ||||
|         activity?.SetTag("stellaops.cli.command", "db fetch"); | ||||
|         activity?.SetTag("stellaops.cli.source", source); | ||||
|         activity?.SetTag("stellaops.cli.stage", stage); | ||||
|         if (!string.IsNullOrWhiteSpace(mode)) | ||||
|         { | ||||
|             activity?.SetTag("stellaops.cli.mode", mode); | ||||
|         } | ||||
|         using var duration = CliMetrics.MeasureCommandDuration("db fetch"); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var jobKind = $"source:{source}:{stage}"; | ||||
|             var parameters = new Dictionary<string, object?>(StringComparer.Ordinal); | ||||
|             if (!string.IsNullOrWhiteSpace(mode)) | ||||
|             { | ||||
|                 parameters["mode"] = mode; | ||||
|             } | ||||
|  | ||||
|             await TriggerJobAsync(client, logger, jobKind, parameters, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogError(ex, "Connector job failed."); | ||||
|             Environment.ExitCode = 1; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             verbosity.MinimumLevel = previousLevel; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public static async Task HandleMergeJobAsync( | ||||
|         IServiceProvider services, | ||||
|         bool verbose, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         await using var scope = services.CreateAsyncScope(); | ||||
|         var client = scope.ServiceProvider.GetRequiredService<IBackendOperationsClient>(); | ||||
|         var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("db-merge"); | ||||
|         var verbosity = scope.ServiceProvider.GetRequiredService<VerbosityState>(); | ||||
|         var previousLevel = verbosity.MinimumLevel; | ||||
|         verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; | ||||
|         using var activity = CliActivitySource.Instance.StartActivity("cli.db.merge", ActivityKind.Client); | ||||
|         activity?.SetTag("stellaops.cli.command", "db merge"); | ||||
|         using var duration = CliMetrics.MeasureCommandDuration("db merge"); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             await TriggerJobAsync(client, logger, "merge:reconcile", new Dictionary<string, object?>(StringComparer.Ordinal), cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogError(ex, "Merge job failed."); | ||||
|             Environment.ExitCode = 1; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             verbosity.MinimumLevel = previousLevel; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public static async Task HandleExportJobAsync( | ||||
|         IServiceProvider services, | ||||
|         string format, | ||||
|         bool delta, | ||||
|         bool verbose, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         await using var scope = services.CreateAsyncScope(); | ||||
|         var client = scope.ServiceProvider.GetRequiredService<IBackendOperationsClient>(); | ||||
|         var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("db-export"); | ||||
|         var verbosity = scope.ServiceProvider.GetRequiredService<VerbosityState>(); | ||||
|         var previousLevel = verbosity.MinimumLevel; | ||||
|         verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; | ||||
|         using var activity = CliActivitySource.Instance.StartActivity("cli.db.export", ActivityKind.Client); | ||||
|         activity?.SetTag("stellaops.cli.command", "db export"); | ||||
|         activity?.SetTag("stellaops.cli.format", format); | ||||
|         activity?.SetTag("stellaops.cli.delta", delta); | ||||
|         using var duration = CliMetrics.MeasureCommandDuration("db export"); | ||||
|  | ||||
|         try | ||||
|         { | ||||
|             var jobKind = format switch | ||||
|             { | ||||
|                 "trivy-db" or "trivy" => "export:trivy-db", | ||||
|                 _ => "export:json" | ||||
|             }; | ||||
|  | ||||
|             var parameters = new Dictionary<string, object?>(StringComparer.Ordinal) | ||||
|             { | ||||
|                 ["delta"] = delta | ||||
|             }; | ||||
|  | ||||
|             await TriggerJobAsync(client, logger, jobKind, parameters, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             logger.LogError(ex, "Export job failed."); | ||||
|             Environment.ExitCode = 1; | ||||
|         } | ||||
|         finally | ||||
|         { | ||||
|             verbosity.MinimumLevel = previousLevel; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static async Task TriggerJobAsync( | ||||
|         IBackendOperationsClient client, | ||||
|         ILogger logger, | ||||
|         string jobKind, | ||||
|         IDictionary<string, object?> parameters, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         JobTriggerResult result = await client.TriggerJobAsync(jobKind, parameters, cancellationToken).ConfigureAwait(false); | ||||
|         if (result.Success) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(result.Location)) | ||||
|             { | ||||
|                 logger.LogInformation("Job accepted. Track status at {Location}.", result.Location); | ||||
|             } | ||||
|             else if (result.Run is not null) | ||||
|             { | ||||
|                 logger.LogInformation("Job accepted. RunId: {RunId} Status: {Status}", result.Run.RunId, result.Run.Status); | ||||
|             } | ||||
|             else | ||||
|             { | ||||
|                 logger.LogInformation("Job accepted."); | ||||
|             } | ||||
|  | ||||
|             Environment.ExitCode = 0; | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             logger.LogError("Job '{JobKind}' failed: {Message}", jobKind, result.Message); | ||||
|             Environment.ExitCode = 1; | ||||
|         } | ||||
|     } | ||||
| } | ||||
							
								
								
									
										77
									
								
								src/StellaOps.Cli/Configuration/CliBootstrapper.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										77
									
								
								src/StellaOps.Cli/Configuration/CliBootstrapper.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,77 @@ | ||||
| using System.Globalization; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using StellaOps.Configuration; | ||||
|  | ||||
| namespace StellaOps.Cli.Configuration; | ||||
|  | ||||
| public static class CliBootstrapper | ||||
| { | ||||
|     public static (StellaOpsCliOptions Options, IConfigurationRoot Configuration) Build(string[] args) | ||||
|     { | ||||
|         var bootstrap = StellaOpsConfigurationBootstrapper.Build<StellaOpsCliOptions>(options => | ||||
|         { | ||||
|             options.BindingSection = "StellaOps"; | ||||
|             options.ConfigureBuilder = builder => | ||||
|             { | ||||
|                 if (args.Length > 0) | ||||
|                 { | ||||
|                     builder.AddCommandLine(args); | ||||
|                 } | ||||
|             }; | ||||
|             options.PostBind = (cliOptions, configuration) => | ||||
|             { | ||||
|                 cliOptions.ApiKey = ResolveWithFallback(cliOptions.ApiKey, configuration, "API_KEY", "StellaOps:ApiKey", "ApiKey"); | ||||
|                 cliOptions.BackendUrl = ResolveWithFallback(cliOptions.BackendUrl, configuration, "STELLAOPS_BACKEND_URL", "StellaOps:BackendUrl", "BackendUrl"); | ||||
|                 cliOptions.ScannerSignaturePublicKeyPath = ResolveWithFallback(cliOptions.ScannerSignaturePublicKeyPath, configuration, "SCANNER_PUBLIC_KEY", "STELLAOPS_SCANNER_PUBLIC_KEY", "StellaOps:ScannerSignaturePublicKeyPath", "ScannerSignaturePublicKeyPath"); | ||||
|  | ||||
|                 cliOptions.ApiKey = cliOptions.ApiKey?.Trim() ?? string.Empty; | ||||
|                 cliOptions.BackendUrl = cliOptions.BackendUrl?.Trim() ?? string.Empty; | ||||
|                 cliOptions.ScannerSignaturePublicKeyPath = cliOptions.ScannerSignaturePublicKeyPath?.Trim() ?? string.Empty; | ||||
|  | ||||
|                 var attemptsRaw = ResolveWithFallback( | ||||
|                     string.Empty, | ||||
|                     configuration, | ||||
|                     "SCANNER_DOWNLOAD_ATTEMPTS", | ||||
|                     "STELLAOPS_SCANNER_DOWNLOAD_ATTEMPTS", | ||||
|                     "StellaOps:ScannerDownloadAttempts", | ||||
|                     "ScannerDownloadAttempts"); | ||||
|  | ||||
|                 if (string.IsNullOrWhiteSpace(attemptsRaw)) | ||||
|                 { | ||||
|                     attemptsRaw = cliOptions.ScannerDownloadAttempts.ToString(CultureInfo.InvariantCulture); | ||||
|                 } | ||||
|  | ||||
|                 if (int.TryParse(attemptsRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedAttempts) && parsedAttempts > 0) | ||||
|                 { | ||||
|                     cliOptions.ScannerDownloadAttempts = parsedAttempts; | ||||
|                 } | ||||
|  | ||||
|                 if (cliOptions.ScannerDownloadAttempts <= 0) | ||||
|                 { | ||||
|                     cliOptions.ScannerDownloadAttempts = 3; | ||||
|                 } | ||||
|             }; | ||||
|         }); | ||||
|  | ||||
|         return (bootstrap.Options, bootstrap.Configuration); | ||||
|     } | ||||
|  | ||||
|     private static string ResolveWithFallback(string currentValue, IConfiguration configuration, params string[] keys) | ||||
|     { | ||||
|         if (!string.IsNullOrWhiteSpace(currentValue)) | ||||
|         { | ||||
|             return currentValue; | ||||
|         } | ||||
|  | ||||
|         foreach (var key in keys) | ||||
|         { | ||||
|             var value = configuration[key]; | ||||
|             if (!string.IsNullOrWhiteSpace(value)) | ||||
|             { | ||||
|                 return value; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return string.Empty; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										18
									
								
								src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										18
									
								
								src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,18 @@ | ||||
| namespace StellaOps.Cli.Configuration; | ||||
|  | ||||
| public sealed class StellaOpsCliOptions | ||||
| { | ||||
|     public string ApiKey { get; set; } = string.Empty; | ||||
|  | ||||
|     public string BackendUrl { get; set; } = string.Empty; | ||||
|  | ||||
|     public string ScannerCacheDirectory { get; set; } = "scanners"; | ||||
|  | ||||
|     public string ResultsDirectory { get; set; } = "results"; | ||||
|  | ||||
|     public string DefaultRunner { get; set; } = "docker"; | ||||
|  | ||||
|     public string ScannerSignaturePublicKeyPath { get; set; } = string.Empty; | ||||
|  | ||||
|     public int ScannerDownloadAttempts { get; set; } = 3; | ||||
| } | ||||
							
								
								
									
										71
									
								
								src/StellaOps.Cli/Program.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										71
									
								
								src/StellaOps.Cli/Program.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,71 @@ | ||||
| using System; | ||||
| using System.CommandLine; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Cli.Commands; | ||||
| using StellaOps.Cli.Configuration; | ||||
| using StellaOps.Cli.Services; | ||||
| using StellaOps.Cli.Telemetry; | ||||
|  | ||||
| namespace StellaOps.Cli; | ||||
|  | ||||
| internal static class Program | ||||
| { | ||||
|     internal static async Task<int> Main(string[] args) | ||||
|     { | ||||
|         var (options, configuration) = CliBootstrapper.Build(args); | ||||
|  | ||||
|         var services = new ServiceCollection(); | ||||
|         services.AddSingleton(configuration); | ||||
|         services.AddSingleton(options); | ||||
|  | ||||
|         var verbosityState = new VerbosityState(); | ||||
|         services.AddSingleton(verbosityState); | ||||
|  | ||||
|         services.AddLogging(builder => | ||||
|         { | ||||
|             builder.ClearProviders(); | ||||
|             builder.AddSimpleConsole(logOptions => | ||||
|             { | ||||
|                 logOptions.TimestampFormat = "HH:mm:ss "; | ||||
|                 logOptions.SingleLine = true; | ||||
|             }); | ||||
|             builder.AddFilter((category, level) => level >= verbosityState.MinimumLevel); | ||||
|         }); | ||||
|  | ||||
|         services.AddHttpClient<IBackendOperationsClient, BackendOperationsClient>(client => | ||||
|         { | ||||
|             client.Timeout = TimeSpan.FromMinutes(5); | ||||
|             if (!string.IsNullOrWhiteSpace(options.BackendUrl) && | ||||
|                 Uri.TryCreate(options.BackendUrl, UriKind.Absolute, out var backendUri)) | ||||
|             { | ||||
|                 client.BaseAddress = backendUri; | ||||
|             } | ||||
|         }); | ||||
|  | ||||
|         services.AddSingleton<IScannerExecutor, ScannerExecutor>(); | ||||
|         services.AddSingleton<IScannerInstaller, ScannerInstaller>(); | ||||
|  | ||||
|         await using var serviceProvider = services.BuildServiceProvider(); | ||||
|         using var cts = new CancellationTokenSource(); | ||||
|         Console.CancelKeyPress += (_, eventArgs) => | ||||
|         { | ||||
|             eventArgs.Cancel = true; | ||||
|             cts.Cancel(); | ||||
|         }; | ||||
|  | ||||
|         var rootCommand = CommandFactory.Create(serviceProvider, options, cts.Token); | ||||
|         var commandConfiguration = new CommandLineConfiguration(rootCommand); | ||||
|         var commandExit = await commandConfiguration.InvokeAsync(args, cts.Token).ConfigureAwait(false); | ||||
|  | ||||
|         var finalExit = Environment.ExitCode != 0 ? Environment.ExitCode : commandExit; | ||||
|         if (cts.IsCancellationRequested && finalExit == 0) | ||||
|         { | ||||
|             finalExit = 130; // Typical POSIX cancellation exit code | ||||
|         } | ||||
|  | ||||
|         return finalExit; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										3
									
								
								src/StellaOps.Cli/Properties/AssemblyInfo.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										3
									
								
								src/StellaOps.Cli/Properties/AssemblyInfo.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,3 @@ | ||||
| using System.Runtime.CompilerServices; | ||||
|  | ||||
| [assembly: InternalsVisibleTo("StellaOps.Cli.Tests")] | ||||
							
								
								
									
										394
									
								
								src/StellaOps.Cli/Services/BackendOperationsClient.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										394
									
								
								src/StellaOps.Cli/Services/BackendOperationsClient.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,394 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using System.Net; | ||||
| using System.Net.Http; | ||||
| using System.Linq; | ||||
| using System.Net.Http.Headers; | ||||
| using System.Net.Http.Json; | ||||
| using System.Security.Cryptography; | ||||
| using System.Text; | ||||
| using System.Text.Json; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Cli.Configuration; | ||||
| using StellaOps.Cli.Services.Models; | ||||
| using StellaOps.Cli.Services.Models.Transport; | ||||
|  | ||||
| namespace StellaOps.Cli.Services; | ||||
|  | ||||
| internal sealed class BackendOperationsClient : IBackendOperationsClient | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); | ||||
|  | ||||
|     private readonly HttpClient _httpClient; | ||||
|     private readonly StellaOpsCliOptions _options; | ||||
|     private readonly ILogger<BackendOperationsClient> _logger; | ||||
|  | ||||
|     public BackendOperationsClient(HttpClient httpClient, StellaOpsCliOptions options, ILogger<BackendOperationsClient> logger) | ||||
|     { | ||||
|         _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); | ||||
|         _options = options ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(_options.BackendUrl) && httpClient.BaseAddress is null) | ||||
|         { | ||||
|             if (Uri.TryCreate(_options.BackendUrl, UriKind.Absolute, out var baseUri)) | ||||
|             { | ||||
|                 httpClient.BaseAddress = baseUri; | ||||
|             } | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public async Task<ScannerArtifactResult> DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken) | ||||
|     { | ||||
|         EnsureBackendConfigured(); | ||||
|  | ||||
|         channel = string.IsNullOrWhiteSpace(channel) ? "stable" : channel.Trim(); | ||||
|         outputPath = ResolveArtifactPath(outputPath, channel); | ||||
|         Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); | ||||
|  | ||||
|         if (!overwrite && File.Exists(outputPath)) | ||||
|         { | ||||
|             var existing = new FileInfo(outputPath); | ||||
|             _logger.LogInformation("Scanner artifact already cached at {Path} ({Size} bytes).", outputPath, existing.Length); | ||||
|             return new ScannerArtifactResult(outputPath, existing.Length, true); | ||||
|         } | ||||
|  | ||||
|         var attempt = 0; | ||||
|         var maxAttempts = Math.Max(1, _options.ScannerDownloadAttempts); | ||||
|  | ||||
|         while (true) | ||||
|         { | ||||
|             attempt++; | ||||
|             try | ||||
|             { | ||||
|                 using var request = CreateRequest(HttpMethod.Get, $"api/scanner/artifacts/{channel}"); | ||||
|                 using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); | ||||
|                 if (!response.IsSuccessStatusCode) | ||||
|                 { | ||||
|                     var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); | ||||
|                     throw new InvalidOperationException(failure); | ||||
|                 } | ||||
|  | ||||
|                 return await ProcessScannerResponseAsync(response, outputPath, channel, verbose, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) when (attempt < maxAttempts) | ||||
|             { | ||||
|                 var backoffSeconds = Math.Pow(2, attempt); | ||||
|                 _logger.LogWarning(ex, "Scanner download attempt {Attempt}/{MaxAttempts} failed. Retrying in {Delay:F0}s...", attempt, maxAttempts, backoffSeconds); | ||||
|                 await Task.Delay(TimeSpan.FromSeconds(backoffSeconds), cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private async Task<ScannerArtifactResult> ProcessScannerResponseAsync(HttpResponseMessage response, string outputPath, string channel, bool verbose, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var tempFile = outputPath + ".tmp"; | ||||
|         await using (var payloadStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) | ||||
|         await using (var fileStream = File.Create(tempFile)) | ||||
|         { | ||||
|             await payloadStream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|  | ||||
|         var expectedDigest = ExtractHeaderValue(response.Headers, "X-StellaOps-Digest"); | ||||
|         var signatureHeader = ExtractHeaderValue(response.Headers, "X-StellaOps-Signature"); | ||||
|  | ||||
|         var digestHex = await ValidateDigestAsync(tempFile, expectedDigest, cancellationToken).ConfigureAwait(false); | ||||
|         await ValidateSignatureAsync(signatureHeader, digestHex, verbose, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         if (verbose) | ||||
|         { | ||||
|             var signatureNote = string.IsNullOrWhiteSpace(signatureHeader) ? "no signature" : "signature validated"; | ||||
|             _logger.LogDebug("Scanner digest sha256:{Digest} ({SignatureNote}).", digestHex, signatureNote); | ||||
|         } | ||||
|  | ||||
|         if (File.Exists(outputPath)) | ||||
|         { | ||||
|             File.Delete(outputPath); | ||||
|         } | ||||
|  | ||||
|         File.Move(tempFile, outputPath); | ||||
|  | ||||
|         PersistMetadata(outputPath, channel, digestHex, signatureHeader, response); | ||||
|  | ||||
|         var downloaded = new FileInfo(outputPath); | ||||
|         _logger.LogInformation("Scanner downloaded to {Path} ({Size} bytes).", outputPath, downloaded.Length); | ||||
|  | ||||
|         return new ScannerArtifactResult(outputPath, downloaded.Length, false); | ||||
|     } | ||||
|  | ||||
|     public async Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) | ||||
|     { | ||||
|         EnsureBackendConfigured(); | ||||
|  | ||||
|         if (!File.Exists(filePath)) | ||||
|         { | ||||
|             throw new FileNotFoundException("Scan result file not found.", filePath); | ||||
|         } | ||||
|  | ||||
|         using var content = new MultipartFormDataContent(); | ||||
|         await using var fileStream = File.OpenRead(filePath); | ||||
|         var streamContent = new StreamContent(fileStream); | ||||
|         streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); | ||||
|         content.Add(streamContent, "file", Path.GetFileName(filePath)); | ||||
|  | ||||
|         var request = CreateRequest(HttpMethod.Post, "api/scanner/results"); | ||||
|         request.Content = content; | ||||
|  | ||||
|         using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|         if (!response.IsSuccessStatusCode) | ||||
|         { | ||||
|             var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); | ||||
|             throw new InvalidOperationException(failure); | ||||
|         } | ||||
|  | ||||
|         _logger.LogInformation("Scan results uploaded from {Path}.", filePath); | ||||
|     } | ||||
|  | ||||
|     public async Task<JobTriggerResult> TriggerJobAsync(string jobKind, IDictionary<string, object?> parameters, CancellationToken cancellationToken) | ||||
|     { | ||||
|         EnsureBackendConfigured(); | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(jobKind)) | ||||
|         { | ||||
|             throw new ArgumentException("Job kind must be provided.", nameof(jobKind)); | ||||
|         } | ||||
|  | ||||
|         var requestBody = new JobTriggerRequest | ||||
|         { | ||||
|             Trigger = "cli", | ||||
|             Parameters = parameters is null ? new Dictionary<string, object?>(StringComparer.Ordinal) : new Dictionary<string, object?>(parameters, StringComparer.Ordinal) | ||||
|         }; | ||||
|  | ||||
|         var request = CreateRequest(HttpMethod.Post, $"jobs/{jobKind}"); | ||||
|         request.Content = JsonContent.Create(requestBody, options: SerializerOptions); | ||||
|  | ||||
|         using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|         if (response.StatusCode == HttpStatusCode.Accepted) | ||||
|         { | ||||
|             JobRunResponse? run = null; | ||||
|             if (response.Content.Headers.ContentLength is > 0) | ||||
|             { | ||||
|                 try | ||||
|                 { | ||||
|                     run = await response.Content.ReadFromJsonAsync<JobRunResponse>(SerializerOptions, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|                 catch (JsonException ex) | ||||
|                 { | ||||
|                     _logger.LogWarning(ex, "Failed to deserialize job run response for job kind {Kind}.", jobKind); | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             var location = response.Headers.Location?.ToString(); | ||||
|             return new JobTriggerResult(true, "Accepted", location, run); | ||||
|         } | ||||
|  | ||||
|         var failureMessage = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); | ||||
|         return new JobTriggerResult(false, failureMessage, null, null); | ||||
|     } | ||||
|  | ||||
|     private HttpRequestMessage CreateRequest(HttpMethod method, string relativeUri) | ||||
|     { | ||||
|         if (!Uri.TryCreate(relativeUri, UriKind.RelativeOrAbsolute, out var requestUri)) | ||||
|         { | ||||
|             throw new InvalidOperationException($"Invalid request URI '{relativeUri}'."); | ||||
|         } | ||||
|  | ||||
|         if (requestUri.IsAbsoluteUri) | ||||
|         { | ||||
|             // Nothing to normalize. | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             requestUri = new Uri(relativeUri.TrimStart('/'), UriKind.Relative); | ||||
|         } | ||||
|  | ||||
|         var request = new HttpRequestMessage(method, requestUri); | ||||
|         if (!string.IsNullOrWhiteSpace(_options.ApiKey)) | ||||
|         { | ||||
|             request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ApiKey); | ||||
|         } | ||||
|  | ||||
|         return request; | ||||
|     } | ||||
|  | ||||
|     private void EnsureBackendConfigured() | ||||
|     { | ||||
|         if (_httpClient.BaseAddress is null) | ||||
|         { | ||||
|             throw new InvalidOperationException("Backend URL is not configured. Provide STELLAOPS_BACKEND_URL or configure appsettings."); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private string ResolveArtifactPath(string outputPath, string channel) | ||||
|     { | ||||
|         if (!string.IsNullOrWhiteSpace(outputPath)) | ||||
|         { | ||||
|             return Path.GetFullPath(outputPath); | ||||
|         } | ||||
|  | ||||
|         var directory = string.IsNullOrWhiteSpace(_options.ScannerCacheDirectory) | ||||
|             ? Directory.GetCurrentDirectory() | ||||
|             : Path.GetFullPath(_options.ScannerCacheDirectory); | ||||
|  | ||||
|         Directory.CreateDirectory(directory); | ||||
|         var fileName = $"stellaops-scanner-{channel}.tar.gz"; | ||||
|         return Path.Combine(directory, fileName); | ||||
|     } | ||||
|  | ||||
|     private async Task<string> CreateFailureMessageAsync(HttpResponseMessage response, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var statusCode = (int)response.StatusCode; | ||||
|         var builder = new StringBuilder(); | ||||
|         builder.Append("Backend request failed with status "); | ||||
|         builder.Append(statusCode); | ||||
|         builder.Append(' '); | ||||
|         builder.Append(response.ReasonPhrase ?? "Unknown"); | ||||
|  | ||||
|         if (response.Content.Headers.ContentLength is > 0) | ||||
|         { | ||||
|             try | ||||
|             { | ||||
|                 var problem = await response.Content.ReadFromJsonAsync<ProblemDocument>(SerializerOptions, cancellationToken).ConfigureAwait(false); | ||||
|                 if (problem is not null) | ||||
|                 { | ||||
|                     if (!string.IsNullOrWhiteSpace(problem.Title)) | ||||
|                     { | ||||
|                         builder.AppendLine().Append(problem.Title); | ||||
|                     } | ||||
|  | ||||
|                     if (!string.IsNullOrWhiteSpace(problem.Detail)) | ||||
|                     { | ||||
|                         builder.AppendLine().Append(problem.Detail); | ||||
|                     } | ||||
|                 } | ||||
|             } | ||||
|             catch (JsonException) | ||||
|             { | ||||
|                 var raw = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); | ||||
|                 if (!string.IsNullOrWhiteSpace(raw)) | ||||
|                 { | ||||
|                     builder.AppendLine().Append(raw); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return builder.ToString(); | ||||
|     } | ||||
|  | ||||
|     private static string? ExtractHeaderValue(HttpResponseHeaders headers, string name) | ||||
|     { | ||||
|         if (headers.TryGetValues(name, out var values)) | ||||
|         { | ||||
|             return values.FirstOrDefault(); | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private async Task<string> ValidateDigestAsync(string filePath, string? expectedDigest, CancellationToken cancellationToken) | ||||
|     { | ||||
|         string digestHex; | ||||
|         await using (var stream = File.OpenRead(filePath)) | ||||
|         { | ||||
|             var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); | ||||
|             digestHex = Convert.ToHexString(hash).ToLowerInvariant(); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(expectedDigest)) | ||||
|         { | ||||
|             var normalized = NormalizeDigest(expectedDigest); | ||||
|             if (!normalized.Equals(digestHex, StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 File.Delete(filePath); | ||||
|                 throw new InvalidOperationException($"Scanner digest mismatch. Expected sha256:{normalized}, calculated sha256:{digestHex}."); | ||||
|             } | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             _logger.LogWarning("Scanner download missing X-StellaOps-Digest header; relying on computed digest only."); | ||||
|         } | ||||
|  | ||||
|         return digestHex; | ||||
|     } | ||||
|  | ||||
|     private static string NormalizeDigest(string digest) | ||||
|     { | ||||
|         if (digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return digest[7..]; | ||||
|         } | ||||
|  | ||||
|         return digest; | ||||
|     } | ||||
|  | ||||
|     private async Task ValidateSignatureAsync(string? signatureHeader, string digestHex, bool verbose, CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(_options.ScannerSignaturePublicKeyPath)) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(signatureHeader)) | ||||
|             { | ||||
|                 _logger.LogDebug("Signature header present but no public key configured; skipping validation."); | ||||
|             } | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(signatureHeader)) | ||||
|         { | ||||
|             throw new InvalidOperationException("Scanner signature missing while a public key is configured."); | ||||
|         } | ||||
|  | ||||
|         var publicKeyPath = Path.GetFullPath(_options.ScannerSignaturePublicKeyPath); | ||||
|         if (!File.Exists(publicKeyPath)) | ||||
|         { | ||||
|             throw new FileNotFoundException("Scanner signature public key not found.", publicKeyPath); | ||||
|         } | ||||
|  | ||||
|         var signatureBytes = Convert.FromBase64String(signatureHeader); | ||||
|         var digestBytes = Convert.FromHexString(digestHex); | ||||
|  | ||||
|         var pem = await File.ReadAllTextAsync(publicKeyPath, cancellationToken).ConfigureAwait(false); | ||||
|         using var rsa = RSA.Create(); | ||||
|         rsa.ImportFromPem(pem); | ||||
|  | ||||
|         var valid = rsa.VerifyHash(digestBytes, signatureBytes, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); | ||||
|         if (!valid) | ||||
|         { | ||||
|             throw new InvalidOperationException("Scanner signature validation failed."); | ||||
|         } | ||||
|  | ||||
|         if (verbose) | ||||
|         { | ||||
|             _logger.LogDebug("Scanner signature validated using key {KeyPath}.", publicKeyPath); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private void PersistMetadata(string outputPath, string channel, string digestHex, string? signatureHeader, HttpResponseMessage response) | ||||
|     { | ||||
|         var metadata = new | ||||
|         { | ||||
|             channel, | ||||
|             digest = $"sha256:{digestHex}", | ||||
|             signature = signatureHeader, | ||||
|             downloadedAt = DateTimeOffset.UtcNow, | ||||
|             source = response.RequestMessage?.RequestUri?.ToString(), | ||||
|             sizeBytes = new FileInfo(outputPath).Length, | ||||
|             headers = new | ||||
|             { | ||||
|                 etag = response.Headers.ETag?.Tag, | ||||
|                 lastModified = response.Content.Headers.LastModified, | ||||
|                 contentType = response.Content.Headers.ContentType?.ToString() | ||||
|             } | ||||
|         }; | ||||
|  | ||||
|         var metadataPath = outputPath + ".metadata.json"; | ||||
|         var json = JsonSerializer.Serialize(metadata, new JsonSerializerOptions | ||||
|         { | ||||
|             WriteIndented = true | ||||
|         }); | ||||
|  | ||||
|         File.WriteAllText(metadataPath, json); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										16
									
								
								src/StellaOps.Cli/Services/IBackendOperationsClient.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										16
									
								
								src/StellaOps.Cli/Services/IBackendOperationsClient.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,16 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Cli.Configuration; | ||||
| using StellaOps.Cli.Services.Models; | ||||
|  | ||||
| namespace StellaOps.Cli.Services; | ||||
|  | ||||
| internal interface IBackendOperationsClient | ||||
| { | ||||
|     Task<ScannerArtifactResult> DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken); | ||||
|  | ||||
|     Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken); | ||||
|  | ||||
|     Task<JobTriggerResult> TriggerJobAsync(string jobKind, IDictionary<string, object?> parameters, CancellationToken cancellationToken); | ||||
| } | ||||
							
								
								
									
										17
									
								
								src/StellaOps.Cli/Services/IScannerExecutor.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										17
									
								
								src/StellaOps.Cli/Services/IScannerExecutor.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,17 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
|  | ||||
| namespace StellaOps.Cli.Services; | ||||
|  | ||||
| internal interface IScannerExecutor | ||||
| { | ||||
|     Task<ScannerExecutionResult> RunAsync( | ||||
|         string runner, | ||||
|         string entry, | ||||
|         string targetDirectory, | ||||
|         string resultsDirectory, | ||||
|         IReadOnlyList<string> arguments, | ||||
|         bool verbose, | ||||
|         CancellationToken cancellationToken); | ||||
| } | ||||
							
								
								
									
										9
									
								
								src/StellaOps.Cli/Services/IScannerInstaller.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										9
									
								
								src/StellaOps.Cli/Services/IScannerInstaller.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,9 @@ | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
|  | ||||
| namespace StellaOps.Cli.Services; | ||||
|  | ||||
| internal interface IScannerInstaller | ||||
| { | ||||
|     Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken); | ||||
| } | ||||
							
								
								
									
										9
									
								
								src/StellaOps.Cli/Services/Models/JobTriggerResult.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										9
									
								
								src/StellaOps.Cli/Services/Models/JobTriggerResult.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,9 @@ | ||||
| using StellaOps.Cli.Services.Models.Transport; | ||||
|  | ||||
| namespace StellaOps.Cli.Services.Models; | ||||
|  | ||||
| internal sealed record JobTriggerResult( | ||||
|     bool Success, | ||||
|     string Message, | ||||
|     string? Location, | ||||
|     JobRunResponse? Run); | ||||
| @@ -0,0 +1,3 @@ | ||||
| namespace StellaOps.Cli.Services.Models; | ||||
|  | ||||
| internal sealed record ScannerArtifactResult(string Path, long SizeBytes, bool FromCache); | ||||
| @@ -0,0 +1,27 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Cli.Services.Models.Transport; | ||||
|  | ||||
| internal sealed class JobRunResponse | ||||
| { | ||||
|     public Guid RunId { get; set; } | ||||
|  | ||||
|     public string Kind { get; set; } = string.Empty; | ||||
|  | ||||
|     public string Status { get; set; } = string.Empty; | ||||
|  | ||||
|     public string Trigger { get; set; } = string.Empty; | ||||
|  | ||||
|     public DateTimeOffset CreatedAt { get; set; } | ||||
|  | ||||
|     public DateTimeOffset? StartedAt { get; set; } | ||||
|  | ||||
|     public DateTimeOffset? CompletedAt { get; set; } | ||||
|  | ||||
|     public string? Error { get; set; } | ||||
|  | ||||
|     public TimeSpan? Duration { get; set; } | ||||
|  | ||||
|     public IReadOnlyDictionary<string, object?> Parameters { get; set; } = new Dictionary<string, object?>(StringComparer.Ordinal); | ||||
| } | ||||
| @@ -0,0 +1,10 @@ | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Cli.Services.Models.Transport; | ||||
|  | ||||
| internal sealed class JobTriggerRequest | ||||
| { | ||||
|     public string Trigger { get; set; } = "cli"; | ||||
|  | ||||
|     public Dictionary<string, object?> Parameters { get; set; } = new(StringComparer.Ordinal); | ||||
| } | ||||
| @@ -0,0 +1,18 @@ | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Cli.Services.Models.Transport; | ||||
|  | ||||
| internal sealed class ProblemDocument | ||||
| { | ||||
|     public string? Type { get; set; } | ||||
|  | ||||
|     public string? Title { get; set; } | ||||
|  | ||||
|     public string? Detail { get; set; } | ||||
|  | ||||
|     public int? Status { get; set; } | ||||
|  | ||||
|     public string? Instance { get; set; } | ||||
|  | ||||
|     public Dictionary<string, object?>? Extensions { get; set; } | ||||
| } | ||||
							
								
								
									
										3
									
								
								src/StellaOps.Cli/Services/ScannerExecutionResult.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										3
									
								
								src/StellaOps.Cli/Services/ScannerExecutionResult.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,3 @@ | ||||
| namespace StellaOps.Cli.Services; | ||||
|  | ||||
| internal sealed record ScannerExecutionResult(int ExitCode, string ResultsPath); | ||||
							
								
								
									
										274
									
								
								src/StellaOps.Cli/Services/ScannerExecutor.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										274
									
								
								src/StellaOps.Cli/Services/ScannerExecutor.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,274 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Diagnostics; | ||||
| using System.IO; | ||||
| using System.Linq; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
|  | ||||
| namespace StellaOps.Cli.Services; | ||||
|  | ||||
| internal sealed class ScannerExecutor : IScannerExecutor | ||||
| { | ||||
|     private readonly ILogger<ScannerExecutor> _logger; | ||||
|  | ||||
|     public ScannerExecutor(ILogger<ScannerExecutor> logger) | ||||
|     { | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task<ScannerExecutionResult> RunAsync( | ||||
|         string runner, | ||||
|         string entry, | ||||
|         string targetDirectory, | ||||
|         string resultsDirectory, | ||||
|         IReadOnlyList<string> arguments, | ||||
|         bool verbose, | ||||
|         CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(targetDirectory)) | ||||
|         { | ||||
|             throw new ArgumentException("Target directory must be provided.", nameof(targetDirectory)); | ||||
|         } | ||||
|  | ||||
|         runner = string.IsNullOrWhiteSpace(runner) ? "docker" : runner.Trim().ToLowerInvariant(); | ||||
|         entry = entry?.Trim() ?? string.Empty; | ||||
|  | ||||
|         var normalizedTarget = Path.GetFullPath(targetDirectory); | ||||
|         if (!Directory.Exists(normalizedTarget)) | ||||
|         { | ||||
|             throw new DirectoryNotFoundException($"Scan target directory '{normalizedTarget}' does not exist."); | ||||
|         } | ||||
|  | ||||
|         resultsDirectory = string.IsNullOrWhiteSpace(resultsDirectory) | ||||
|             ? Path.Combine(Directory.GetCurrentDirectory(), "scan-results") | ||||
|             : Path.GetFullPath(resultsDirectory); | ||||
|  | ||||
|         Directory.CreateDirectory(resultsDirectory); | ||||
|         var executionTimestamp = DateTimeOffset.UtcNow; | ||||
|         var baselineFiles = Directory.GetFiles(resultsDirectory, "*", SearchOption.AllDirectories); | ||||
|         var baseline = new HashSet<string>(baselineFiles, StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         var startInfo = BuildProcessStartInfo(runner, entry, normalizedTarget, resultsDirectory, arguments); | ||||
|         using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; | ||||
|  | ||||
|         var stdout = new List<string>(); | ||||
|         var stderr = new List<string>(); | ||||
|  | ||||
|         process.OutputDataReceived += (_, args) => | ||||
|         { | ||||
|             if (args.Data is null) | ||||
|             { | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             stdout.Add(args.Data); | ||||
|             if (verbose) | ||||
|             { | ||||
|                 _logger.LogInformation("[scan] {Line}", args.Data); | ||||
|             } | ||||
|         }; | ||||
|  | ||||
|         process.ErrorDataReceived += (_, args) => | ||||
|         { | ||||
|             if (args.Data is null) | ||||
|             { | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             stderr.Add(args.Data); | ||||
|             _logger.LogError("[scan] {Line}", args.Data); | ||||
|         }; | ||||
|  | ||||
|         _logger.LogInformation("Launching scanner via {Runner} (entry: {Entry})...", runner, entry); | ||||
|         if (!process.Start()) | ||||
|         { | ||||
|             throw new InvalidOperationException("Failed to start scanner process."); | ||||
|         } | ||||
|  | ||||
|         process.BeginOutputReadLine(); | ||||
|         process.BeginErrorReadLine(); | ||||
|  | ||||
|         await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         if (process.ExitCode == 0) | ||||
|         { | ||||
|             _logger.LogInformation("Scanner completed successfully."); | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             _logger.LogWarning("Scanner exited with code {Code}.", process.ExitCode); | ||||
|         } | ||||
|  | ||||
|         var resultsPath = ResolveResultsPath(resultsDirectory, executionTimestamp, baseline); | ||||
|         if (string.IsNullOrWhiteSpace(resultsPath)) | ||||
|         { | ||||
|             resultsPath = CreatePlaceholderResult(resultsDirectory); | ||||
|         } | ||||
|  | ||||
|         return new ScannerExecutionResult(process.ExitCode, resultsPath); | ||||
|     } | ||||
|  | ||||
|     private ProcessStartInfo BuildProcessStartInfo( | ||||
|         string runner, | ||||
|         string entry, | ||||
|         string targetDirectory, | ||||
|         string resultsDirectory, | ||||
|         IReadOnlyList<string> args) | ||||
|     { | ||||
|         return runner switch | ||||
|         { | ||||
|             "self" or "native" => BuildNativeStartInfo(entry, args), | ||||
|             "dotnet" => BuildDotNetStartInfo(entry, args), | ||||
|             "docker" => BuildDockerStartInfo(entry, targetDirectory, resultsDirectory, args), | ||||
|             _ => BuildCustomRunnerStartInfo(runner, entry, args) | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static ProcessStartInfo BuildNativeStartInfo(string binaryPath, IReadOnlyList<string> args) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(binaryPath) || !File.Exists(binaryPath)) | ||||
|         { | ||||
|             throw new FileNotFoundException("Scanner entrypoint not found.", binaryPath); | ||||
|         } | ||||
|  | ||||
|         var startInfo = new ProcessStartInfo | ||||
|         { | ||||
|             FileName = binaryPath, | ||||
|             WorkingDirectory = Directory.GetCurrentDirectory() | ||||
|         }; | ||||
|  | ||||
|         foreach (var argument in args) | ||||
|         { | ||||
|             startInfo.ArgumentList.Add(argument); | ||||
|         } | ||||
|  | ||||
|         startInfo.RedirectStandardError = true; | ||||
|         startInfo.RedirectStandardOutput = true; | ||||
|         startInfo.UseShellExecute = false; | ||||
|  | ||||
|         return startInfo; | ||||
|     } | ||||
|  | ||||
|     private static ProcessStartInfo BuildDotNetStartInfo(string binaryPath, IReadOnlyList<string> args) | ||||
|     { | ||||
|         var startInfo = new ProcessStartInfo | ||||
|         { | ||||
|             FileName = "dotnet", | ||||
|             WorkingDirectory = Directory.GetCurrentDirectory() | ||||
|         }; | ||||
|  | ||||
|         startInfo.ArgumentList.Add(binaryPath); | ||||
|         foreach (var argument in args) | ||||
|         { | ||||
|             startInfo.ArgumentList.Add(argument); | ||||
|         } | ||||
|  | ||||
|         startInfo.RedirectStandardError = true; | ||||
|         startInfo.RedirectStandardOutput = true; | ||||
|         startInfo.UseShellExecute = false; | ||||
|  | ||||
|         return startInfo; | ||||
|     } | ||||
|  | ||||
|     private static ProcessStartInfo BuildDockerStartInfo(string image, string targetDirectory, string resultsDirectory, IReadOnlyList<string> args) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(image)) | ||||
|         { | ||||
|             throw new ArgumentException("Docker image must be provided when runner is 'docker'.", nameof(image)); | ||||
|         } | ||||
|  | ||||
|         var cwd = Directory.GetCurrentDirectory(); | ||||
|  | ||||
|         var startInfo = new ProcessStartInfo | ||||
|         { | ||||
|             FileName = "docker", | ||||
|             WorkingDirectory = cwd | ||||
|         }; | ||||
|  | ||||
|         startInfo.ArgumentList.Add("run"); | ||||
|         startInfo.ArgumentList.Add("--rm"); | ||||
|         startInfo.ArgumentList.Add("-v"); | ||||
|         startInfo.ArgumentList.Add($"{cwd}:{cwd}"); | ||||
|         startInfo.ArgumentList.Add("-v"); | ||||
|         startInfo.ArgumentList.Add($"{targetDirectory}:/scan-target:ro"); | ||||
|         startInfo.ArgumentList.Add("-v"); | ||||
|         startInfo.ArgumentList.Add($"{resultsDirectory}:/scan-results"); | ||||
|         startInfo.ArgumentList.Add("-w"); | ||||
|         startInfo.ArgumentList.Add(cwd); | ||||
|         startInfo.ArgumentList.Add(image); | ||||
|         startInfo.ArgumentList.Add("--target"); | ||||
|         startInfo.ArgumentList.Add("/scan-target"); | ||||
|         startInfo.ArgumentList.Add("--output"); | ||||
|         startInfo.ArgumentList.Add("/scan-results/scan.json"); | ||||
|  | ||||
|         foreach (var argument in args) | ||||
|         { | ||||
|             startInfo.ArgumentList.Add(argument); | ||||
|         } | ||||
|  | ||||
|         startInfo.RedirectStandardError = true; | ||||
|         startInfo.RedirectStandardOutput = true; | ||||
|         startInfo.UseShellExecute = false; | ||||
|  | ||||
|         return startInfo; | ||||
|     } | ||||
|  | ||||
|     private static ProcessStartInfo BuildCustomRunnerStartInfo(string runner, string entry, IReadOnlyList<string> args) | ||||
|     { | ||||
|         var startInfo = new ProcessStartInfo | ||||
|         { | ||||
|             FileName = runner, | ||||
|             WorkingDirectory = Directory.GetCurrentDirectory() | ||||
|         }; | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(entry)) | ||||
|         { | ||||
|             startInfo.ArgumentList.Add(entry); | ||||
|         } | ||||
|  | ||||
|         foreach (var argument in args) | ||||
|         { | ||||
|             startInfo.ArgumentList.Add(argument); | ||||
|         } | ||||
|  | ||||
|         startInfo.RedirectStandardError = true; | ||||
|         startInfo.RedirectStandardOutput = true; | ||||
|         startInfo.UseShellExecute = false; | ||||
|  | ||||
|         return startInfo; | ||||
|     } | ||||
|  | ||||
|     private static string ResolveResultsPath(string resultsDirectory, DateTimeOffset startTimestamp, HashSet<string> baseline) | ||||
|     { | ||||
|         var candidates = Directory.GetFiles(resultsDirectory, "*", SearchOption.AllDirectories); | ||||
|         string? newest = null; | ||||
|         DateTimeOffset newestTimestamp = startTimestamp; | ||||
|  | ||||
|         foreach (var candidate in candidates) | ||||
|         { | ||||
|             if (baseline.Contains(candidate)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var info = new FileInfo(candidate); | ||||
|             if (info.LastWriteTimeUtc >= newestTimestamp) | ||||
|             { | ||||
|                 newestTimestamp = info.LastWriteTimeUtc; | ||||
|                 newest = candidate; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return newest ?? string.Empty; | ||||
|     } | ||||
|  | ||||
|     private static string CreatePlaceholderResult(string resultsDirectory) | ||||
|     { | ||||
|         var fileName = $"scan-{DateTimeOffset.UtcNow:yyyyMMddHHmmss}.json"; | ||||
|         var path = Path.Combine(resultsDirectory, fileName); | ||||
|         File.WriteAllText(path, "{\"status\":\"placeholder\"}"); | ||||
|         return path; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										79
									
								
								src/StellaOps.Cli/Services/ScannerInstaller.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										79
									
								
								src/StellaOps.Cli/Services/ScannerInstaller.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,79 @@ | ||||
| using System; | ||||
| using System.Diagnostics; | ||||
| using System.IO; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
|  | ||||
| namespace StellaOps.Cli.Services; | ||||
|  | ||||
| internal sealed class ScannerInstaller : IScannerInstaller | ||||
| { | ||||
|     private readonly ILogger<ScannerInstaller> _logger; | ||||
|  | ||||
|     public ScannerInstaller(ILogger<ScannerInstaller> logger) | ||||
|     { | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(artifactPath) || !File.Exists(artifactPath)) | ||||
|         { | ||||
|             throw new FileNotFoundException("Scanner artifact not found for installation.", artifactPath); | ||||
|         } | ||||
|  | ||||
|         // Current implementation assumes docker-based scanner bundle. | ||||
|         var processInfo = new ProcessStartInfo | ||||
|         { | ||||
|             FileName = "docker", | ||||
|             ArgumentList = { "load", "-i", artifactPath }, | ||||
|             RedirectStandardOutput = true, | ||||
|             RedirectStandardError = true, | ||||
|             UseShellExecute = false | ||||
|         }; | ||||
|  | ||||
|         using var process = new Process { StartInfo = processInfo, EnableRaisingEvents = true }; | ||||
|  | ||||
|         process.OutputDataReceived += (_, args) => | ||||
|         { | ||||
|             if (args.Data is null) | ||||
|             { | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             if (verbose) | ||||
|             { | ||||
|                 _logger.LogInformation("[install] {Line}", args.Data); | ||||
|             } | ||||
|         }; | ||||
|  | ||||
|         process.ErrorDataReceived += (_, args) => | ||||
|         { | ||||
|             if (args.Data is null) | ||||
|             { | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             _logger.LogError("[install] {Line}", args.Data); | ||||
|         }; | ||||
|  | ||||
|         _logger.LogInformation("Installing scanner container from {Path}...", artifactPath); | ||||
|         if (!process.Start()) | ||||
|         { | ||||
|             throw new InvalidOperationException("Failed to start container installation process."); | ||||
|         } | ||||
|  | ||||
|         process.BeginOutputReadLine(); | ||||
|         process.BeginErrorReadLine(); | ||||
|  | ||||
|         await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         if (process.ExitCode != 0) | ||||
|         { | ||||
|             throw new InvalidOperationException($"Container installation failed with exit code {process.ExitCode}."); | ||||
|         } | ||||
|  | ||||
|         _logger.LogInformation("Scanner container installed successfully."); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										41
									
								
								src/StellaOps.Cli/StellaOps.Cli.csproj
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										41
									
								
								src/StellaOps.Cli/StellaOps.Cli.csproj
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,41 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|  | ||||
|   <PropertyGroup> | ||||
|     <OutputType>Exe</OutputType> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" /> | ||||
|     <PackageReference Include="NetEscapades.Configuration.Yaml" Version="2.1.0" /> | ||||
|     <PackageReference Include="System.CommandLine" Version="2.0.0-beta5.25306.1" /> | ||||
|   </ItemGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <Content Include="appsettings.json"> | ||||
|       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||||
|     </Content> | ||||
|     <Content Include="appsettings.local.json" Condition="Exists('appsettings.local.json')"> | ||||
|       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||||
|     </Content> | ||||
|     <Content Include="appsettings.yaml" Condition="Exists('appsettings.yaml')"> | ||||
|       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||||
|     </Content> | ||||
|     <Content Include="appsettings.local.yaml" Condition="Exists('appsettings.local.yaml')"> | ||||
|       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||||
|     </Content> | ||||
|   </ItemGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="..\StellaOps.Configuration\StellaOps.Configuration.csproj" /> | ||||
|   </ItemGroup> | ||||
|  | ||||
| </Project> | ||||
							
								
								
									
										9
									
								
								src/StellaOps.Cli/TASKS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										9
									
								
								src/StellaOps.Cli/TASKS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,9 @@ | ||||
| # TASKS | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |Bootstrap configuration fallback (env → appsettings{{.json/.yaml}})|DevEx/CLI|Core|**DONE** – CLI loads `API_KEY`/`STELLAOPS_BACKEND_URL` from environment or local settings, defaulting to empty strings when unset.| | ||||
| |Introduce command host & routing skeleton|DevEx/CLI|Configuration|**DONE** – System.CommandLine (v2.0.0-beta5) router stitched with `scanner`, `scan`, `db`, and `config` verbs.| | ||||
| |Scanner artifact download/install commands|Ops Integrator|Backend contracts|**DONE** – `scanner download` caches bundles, validates SHA-256 (plus optional RSA signature), installs via `docker load`, persists metadata, and retries with exponential backoff.| | ||||
| |Scan execution & result upload workflow|Ops Integrator, QA|Scanner cmd|**DONE** – `scan run` drives container scans against directories, emits artefacts in `ResultsDirectory`, auto-uploads on success, and `scan upload` covers manual retries.| | ||||
| |Feedser DB operations passthrough|DevEx/CLI|Backend, Feedser APIs|**DONE** – `db fetch|merge|export` trigger `/jobs/*` endpoints with parameter binding and consistent exit codes.| | ||||
| |CLI observability & tests|QA|Command host|**DONE** – Added console logging defaults & configuration bootstrap tests; future metrics hooks tracked separately.| | ||||
							
								
								
									
										8
									
								
								src/StellaOps.Cli/Telemetry/CliActivitySource.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										8
									
								
								src/StellaOps.Cli/Telemetry/CliActivitySource.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,8 @@ | ||||
| using System.Diagnostics; | ||||
|  | ||||
| namespace StellaOps.Cli.Telemetry; | ||||
|  | ||||
| internal static class CliActivitySource | ||||
| { | ||||
|     public static readonly ActivitySource Instance = new("StellaOps.Cli"); | ||||
| } | ||||
							
								
								
									
										62
									
								
								src/StellaOps.Cli/Telemetry/CliMetrics.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										62
									
								
								src/StellaOps.Cli/Telemetry/CliMetrics.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,62 @@ | ||||
| using System; | ||||
| using System.Diagnostics.Metrics; | ||||
|  | ||||
| namespace StellaOps.Cli.Telemetry; | ||||
|  | ||||
| internal static class CliMetrics | ||||
| { | ||||
|     private static readonly Meter Meter = new("StellaOps.Cli", "1.0.0"); | ||||
|  | ||||
|     private static readonly Counter<long> ScannerDownloadCounter = Meter.CreateCounter<long>("stellaops.cli.scanner.download.count"); | ||||
|     private static readonly Counter<long> ScannerInstallCounter = Meter.CreateCounter<long>("stellaops.cli.scanner.install.count"); | ||||
|     private static readonly Counter<long> ScanRunCounter = Meter.CreateCounter<long>("stellaops.cli.scan.run.count"); | ||||
|     private static readonly Histogram<double> CommandDurationHistogram = Meter.CreateHistogram<double>("stellaops.cli.command.duration.ms"); | ||||
|  | ||||
|     public static void RecordScannerDownload(string channel, bool fromCache) | ||||
|         => ScannerDownloadCounter.Add(1, new KeyValuePair<string, object?>[] | ||||
|         { | ||||
|             new("channel", channel), | ||||
|             new("cache", fromCache ? "hit" : "miss") | ||||
|         }); | ||||
|  | ||||
|     public static void RecordScannerInstall(string channel) | ||||
|         => ScannerInstallCounter.Add(1, new KeyValuePair<string, object?>[] { new("channel", channel) }); | ||||
|  | ||||
|     public static void RecordScanRun(string runner, int exitCode) | ||||
|         => ScanRunCounter.Add(1, new KeyValuePair<string, object?>[] | ||||
|         { | ||||
|             new("runner", runner), | ||||
|             new("exit_code", exitCode) | ||||
|         }); | ||||
|  | ||||
|     public static IDisposable MeasureCommandDuration(string command) | ||||
|     { | ||||
|         var start = DateTime.UtcNow; | ||||
|         return new DurationScope(command, start); | ||||
|     } | ||||
|  | ||||
|     private sealed class DurationScope : IDisposable | ||||
|     { | ||||
|         private readonly string _command; | ||||
|         private readonly DateTime _start; | ||||
|         private bool _disposed; | ||||
|  | ||||
|         public DurationScope(string command, DateTime start) | ||||
|         { | ||||
|             _command = command; | ||||
|             _start = start; | ||||
|         } | ||||
|  | ||||
|         public void Dispose() | ||||
|         { | ||||
|             if (_disposed) | ||||
|             { | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             _disposed = true; | ||||
|             var elapsed = (DateTime.UtcNow - _start).TotalMilliseconds; | ||||
|             CommandDurationHistogram.Record(elapsed, new KeyValuePair<string, object?>[] { new("command", _command) }); | ||||
|         } | ||||
|     } | ||||
| } | ||||
							
								
								
									
										8
									
								
								src/StellaOps.Cli/Telemetry/VerbosityState.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										8
									
								
								src/StellaOps.Cli/Telemetry/VerbosityState.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,8 @@ | ||||
| using Microsoft.Extensions.Logging; | ||||
|  | ||||
| namespace StellaOps.Cli.Telemetry; | ||||
|  | ||||
| internal sealed class VerbosityState | ||||
| { | ||||
|     public LogLevel MinimumLevel { get; set; } = LogLevel.Information; | ||||
| } | ||||
							
								
								
									
										11
									
								
								src/StellaOps.Cli/appsettings.json
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										11
									
								
								src/StellaOps.Cli/appsettings.json
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,11 @@ | ||||
| { | ||||
|   "StellaOps": { | ||||
|     "ApiKey": "", | ||||
|     "BackendUrl": "", | ||||
|     "ScannerCacheDirectory": "scanners", | ||||
|     "ResultsDirectory": "results", | ||||
|     "DefaultRunner": "dotnet", | ||||
|     "ScannerSignaturePublicKeyPath": "", | ||||
|     "ScannerDownloadAttempts": 3 | ||||
|   } | ||||
| } | ||||
							
								
								
									
										18
									
								
								src/StellaOps.Configuration/StellaOps.Configuration.csproj
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										18
									
								
								src/StellaOps.Configuration/StellaOps.Configuration.csproj
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,18 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|  | ||||
|   <PropertyGroup> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" /> | ||||
|     <PackageReference Include="NetEscapades.Configuration.Yaml" Version="2.1.0" /> | ||||
|   </ItemGroup> | ||||
|  | ||||
| </Project> | ||||
							
								
								
									
										64
									
								
								src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										64
									
								
								src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,64 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using Microsoft.Extensions.Configuration; | ||||
|  | ||||
| namespace StellaOps.Configuration; | ||||
|  | ||||
| public sealed class StellaOpsBootstrapOptions<TOptions> | ||||
|     where TOptions : class, new() | ||||
| { | ||||
|     public StellaOpsBootstrapOptions() | ||||
|     { | ||||
|         ConfigurationOptions = new StellaOpsConfigurationOptions(); | ||||
|     } | ||||
|  | ||||
|     internal StellaOpsConfigurationOptions ConfigurationOptions { get; } | ||||
|  | ||||
|     public string? BasePath | ||||
|     { | ||||
|         get => ConfigurationOptions.BasePath; | ||||
|         set => ConfigurationOptions.BasePath = value; | ||||
|     } | ||||
|  | ||||
|     public bool IncludeJsonFiles | ||||
|     { | ||||
|         get => ConfigurationOptions.IncludeJsonFiles; | ||||
|         set => ConfigurationOptions.IncludeJsonFiles = value; | ||||
|     } | ||||
|  | ||||
|     public bool IncludeYamlFiles | ||||
|     { | ||||
|         get => ConfigurationOptions.IncludeYamlFiles; | ||||
|         set => ConfigurationOptions.IncludeYamlFiles = value; | ||||
|     } | ||||
|  | ||||
|     public bool IncludeEnvironmentVariables | ||||
|     { | ||||
|         get => ConfigurationOptions.IncludeEnvironmentVariables; | ||||
|         set => ConfigurationOptions.IncludeEnvironmentVariables = value; | ||||
|     } | ||||
|  | ||||
|     public string? EnvironmentPrefix | ||||
|     { | ||||
|         get => ConfigurationOptions.EnvironmentPrefix; | ||||
|         set => ConfigurationOptions.EnvironmentPrefix = value; | ||||
|     } | ||||
|  | ||||
|     public IList<JsonConfigurationFile> JsonFiles => ConfigurationOptions.JsonFiles; | ||||
|  | ||||
|     public IList<YamlConfigurationFile> YamlFiles => ConfigurationOptions.YamlFiles; | ||||
|  | ||||
|     public string? BindingSection | ||||
|     { | ||||
|         get => ConfigurationOptions.BindingSection; | ||||
|         set => ConfigurationOptions.BindingSection = value; | ||||
|     } | ||||
|  | ||||
|     public Action<IConfigurationBuilder>? ConfigureBuilder | ||||
|     { | ||||
|         get => ConfigurationOptions.ConfigureBuilder; | ||||
|         set => ConfigurationOptions.ConfigureBuilder = value; | ||||
|     } | ||||
|  | ||||
|     public Action<TOptions, IConfiguration>? PostBind { get; set; } | ||||
| } | ||||
| @@ -0,0 +1,106 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using NetEscapades.Configuration.Yaml; | ||||
|  | ||||
| namespace StellaOps.Configuration; | ||||
|  | ||||
| public static class StellaOpsConfigurationBootstrapper | ||||
| { | ||||
|     public static StellaOpsConfigurationContext<TOptions> Build<TOptions>( | ||||
|         Action<StellaOpsBootstrapOptions<TOptions>>? configure = null) | ||||
|         where TOptions : class, new() | ||||
|     { | ||||
|         var bootstrapOptions = new StellaOpsBootstrapOptions<TOptions>(); | ||||
|         configure?.Invoke(bootstrapOptions); | ||||
|  | ||||
|         var configurationOptions = bootstrapOptions.ConfigurationOptions; | ||||
|         var builder = new ConfigurationBuilder(); | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(configurationOptions.BasePath)) | ||||
|         { | ||||
|             builder.SetBasePath(configurationOptions.BasePath!); | ||||
|         } | ||||
|  | ||||
|         if (configurationOptions.IncludeJsonFiles) | ||||
|         { | ||||
|             foreach (var file in configurationOptions.JsonFiles) | ||||
|             { | ||||
|                 builder.AddJsonFile(file.Path, optional: file.Optional, reloadOnChange: file.ReloadOnChange); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (configurationOptions.IncludeYamlFiles) | ||||
|         { | ||||
|             foreach (var file in configurationOptions.YamlFiles) | ||||
|             { | ||||
|                 builder.AddYamlFile(file.Path, optional: file.Optional); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         configurationOptions.ConfigureBuilder?.Invoke(builder); | ||||
|  | ||||
|         if (configurationOptions.IncludeEnvironmentVariables) | ||||
|         { | ||||
|             builder.AddEnvironmentVariables(configurationOptions.EnvironmentPrefix); | ||||
|         } | ||||
|  | ||||
|         var configuration = builder.Build(); | ||||
|  | ||||
|         IConfiguration bindingSource; | ||||
|         if (string.IsNullOrWhiteSpace(configurationOptions.BindingSection)) | ||||
|         { | ||||
|             bindingSource = configuration; | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             bindingSource = configuration.GetSection(configurationOptions.BindingSection!); | ||||
|         } | ||||
|  | ||||
|         var options = new TOptions(); | ||||
|         bindingSource.Bind(options); | ||||
|  | ||||
|         bootstrapOptions.PostBind?.Invoke(options, configuration); | ||||
|  | ||||
|         return new StellaOpsConfigurationContext<TOptions>(configuration, options); | ||||
|     } | ||||
|  | ||||
|     public static IConfigurationBuilder AddStellaOpsDefaults( | ||||
|         this IConfigurationBuilder builder, | ||||
|         Action<StellaOpsConfigurationOptions>? configure = null) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(builder); | ||||
|  | ||||
|         var options = new StellaOpsConfigurationOptions(); | ||||
|         configure?.Invoke(options); | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(options.BasePath)) | ||||
|         { | ||||
|             builder.SetBasePath(options.BasePath!); | ||||
|         } | ||||
|  | ||||
|         if (options.IncludeJsonFiles) | ||||
|         { | ||||
|             foreach (var file in options.JsonFiles) | ||||
|             { | ||||
|                 builder.AddJsonFile(file.Path, optional: file.Optional, reloadOnChange: file.ReloadOnChange); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (options.IncludeYamlFiles) | ||||
|         { | ||||
|             foreach (var file in options.YamlFiles) | ||||
|             { | ||||
|                 builder.AddYamlFile(file.Path, optional: file.Optional); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         options.ConfigureBuilder?.Invoke(builder); | ||||
|  | ||||
|         if (options.IncludeEnvironmentVariables) | ||||
|         { | ||||
|             builder.AddEnvironmentVariables(options.EnvironmentPrefix); | ||||
|         } | ||||
|  | ||||
|         return builder; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										18
									
								
								src/StellaOps.Configuration/StellaOpsConfigurationContext.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										18
									
								
								src/StellaOps.Configuration/StellaOpsConfigurationContext.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,18 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
|  | ||||
| namespace StellaOps.Configuration; | ||||
|  | ||||
| public sealed class StellaOpsConfigurationContext<TOptions> | ||||
|     where TOptions : class, new() | ||||
| { | ||||
|     public StellaOpsConfigurationContext(IConfigurationRoot configuration, TOptions options) | ||||
|     { | ||||
|         Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); | ||||
|         Options = options ?? throw new ArgumentNullException(nameof(options)); | ||||
|     } | ||||
|  | ||||
|     public IConfigurationRoot Configuration { get; } | ||||
|  | ||||
|     public TOptions Options { get; } | ||||
| } | ||||
							
								
								
									
										49
									
								
								src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										49
									
								
								src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,49 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using Microsoft.Extensions.Configuration; | ||||
|  | ||||
| namespace StellaOps.Configuration; | ||||
|  | ||||
| /// <summary> | ||||
| /// Defines how default StellaOps configuration sources are composed. | ||||
| /// </summary> | ||||
| public sealed class StellaOpsConfigurationOptions | ||||
| { | ||||
|     public string? BasePath { get; set; } = Directory.GetCurrentDirectory(); | ||||
|  | ||||
|     public bool IncludeJsonFiles { get; set; } = true; | ||||
|  | ||||
|     public bool IncludeYamlFiles { get; set; } = true; | ||||
|  | ||||
|     public bool IncludeEnvironmentVariables { get; set; } = true; | ||||
|  | ||||
|     public string? EnvironmentPrefix { get; set; } | ||||
|  | ||||
|     public IList<JsonConfigurationFile> JsonFiles { get; } = new List<JsonConfigurationFile> | ||||
|     { | ||||
|         new("appsettings.json", true, false), | ||||
|         new("appsettings.local.json", true, false) | ||||
|     }; | ||||
|  | ||||
|     public IList<YamlConfigurationFile> YamlFiles { get; } = new List<YamlConfigurationFile> | ||||
|     { | ||||
|         new("appsettings.yaml", true), | ||||
|         new("appsettings.local.yaml", true) | ||||
|     }; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Optional hook to register additional configuration sources (e.g. module-specific YAML files). | ||||
|     /// </summary> | ||||
|     public Action<IConfigurationBuilder>? ConfigureBuilder { get; set; } | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Optional configuration section name used when binding strongly typed options. | ||||
|     /// Null or empty indicates the root. | ||||
|     /// </summary> | ||||
|     public string? BindingSection { get; set; } | ||||
| } | ||||
|  | ||||
| public sealed record JsonConfigurationFile(string Path, bool Optional = true, bool ReloadOnChange = false); | ||||
|  | ||||
| public sealed record YamlConfigurationFile(string Path, bool Optional = true); | ||||
							
								
								
									
										26
									
								
								src/StellaOps.Configuration/StellaOpsOptionsBinder.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										26
									
								
								src/StellaOps.Configuration/StellaOpsOptionsBinder.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,26 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
|  | ||||
| namespace StellaOps.Configuration; | ||||
|  | ||||
| public static class StellaOpsOptionsBinder | ||||
| { | ||||
|     public static TOptions BindOptions<TOptions>( | ||||
|         this IConfiguration configuration, | ||||
|         string? section = null, | ||||
|         Action<TOptions, IConfiguration>? postConfigure = null) | ||||
|         where TOptions : class, new() | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         var options = new TOptions(); | ||||
|         var bindingSource = string.IsNullOrWhiteSpace(section) | ||||
|             ? configuration | ||||
|             : configuration.GetSection(section); | ||||
|  | ||||
|         bindingSource.Bind(options); | ||||
|         postConfigure?.Invoke(options, configuration); | ||||
|  | ||||
|         return options; | ||||
|     } | ||||
| } | ||||
| @@ -1,6 +1,7 @@ | ||||
| using System; | ||||
| using System.Globalization; | ||||
| using System.IO; | ||||
| using System.Linq; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Storage.Mongo.Advisories; | ||||
| @@ -64,7 +65,13 @@ public sealed class JsonFeedExporter : IFeedExporter | ||||
|             result.AdvisoryCount, | ||||
|             digest); | ||||
|  | ||||
|         if (existingState is not null && string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal)) | ||||
|         var manifest = result.Files | ||||
|             .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest)) | ||||
|             .ToArray(); | ||||
|  | ||||
|         if (existingState is not null | ||||
|             && existingState.Files.Count > 0 | ||||
|             && string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal)) | ||||
|         { | ||||
|             _logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId); | ||||
|             TryDeleteDirectory(result.ExportDirectory); | ||||
| @@ -90,6 +97,7 @@ public sealed class JsonFeedExporter : IFeedExporter | ||||
|             targetRepository: _options.TargetRepository, | ||||
|             exporterVersion: _exporterVersion, | ||||
|             resetBaseline: resetBaseline, | ||||
|             manifest: manifest, | ||||
|             cancellationToken: cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false); | ||||
|   | ||||
| @@ -10,19 +10,22 @@ public sealed class TrivyDbExportPlannerTests | ||||
|     public void CreatePlan_ReturnsFullWhenStateMissing() | ||||
|     { | ||||
|         var planner = new TrivyDbExportPlanner(); | ||||
|         var plan = planner.CreatePlan(existingState: null, treeDigest: "sha256:abcd"); | ||||
|         var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; | ||||
|         var plan = planner.CreatePlan(existingState: null, treeDigest: "sha256:abcd", manifest); | ||||
|  | ||||
|         Assert.Equal(TrivyDbExportMode.Full, plan.Mode); | ||||
|         Assert.Equal("sha256:abcd", plan.TreeDigest); | ||||
|         Assert.Null(plan.BaseExportId); | ||||
|         Assert.Null(plan.BaseManifestDigest); | ||||
|         Assert.True(plan.ResetBaseline); | ||||
|         Assert.Equal(manifest, plan.Manifest); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public void CreatePlan_ReturnsSkipWhenCursorMatches() | ||||
|     { | ||||
|         var planner = new TrivyDbExportPlanner(); | ||||
|         var existingManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; | ||||
|         var state = new ExportStateRecord( | ||||
|             Id: TrivyDbFeedExporter.ExporterId, | ||||
|             BaseExportId: "20240810T000000Z", | ||||
| @@ -32,21 +35,25 @@ public sealed class TrivyDbExportPlannerTests | ||||
|             ExportCursor: "sha256:unchanged", | ||||
|             TargetRepository: "feedser/trivy", | ||||
|             ExporterVersion: "1.0", | ||||
|             UpdatedAt: DateTimeOffset.UtcNow); | ||||
|             UpdatedAt: DateTimeOffset.UtcNow, | ||||
|             Files: existingManifest); | ||||
|  | ||||
|         var plan = planner.CreatePlan(state, "sha256:unchanged"); | ||||
|         var plan = planner.CreatePlan(state, "sha256:unchanged", existingManifest); | ||||
|  | ||||
|         Assert.Equal(TrivyDbExportMode.Skip, plan.Mode); | ||||
|         Assert.Equal("sha256:unchanged", plan.TreeDigest); | ||||
|         Assert.Equal("20240810T000000Z", plan.BaseExportId); | ||||
|         Assert.Equal("sha256:base", plan.BaseManifestDigest); | ||||
|         Assert.False(plan.ResetBaseline); | ||||
|         Assert.Empty(plan.ChangedFiles); | ||||
|         Assert.Empty(plan.RemovedPaths); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public void CreatePlan_ReturnsFullWhenCursorDiffers() | ||||
|     { | ||||
|         var planner = new TrivyDbExportPlanner(); | ||||
|         var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; | ||||
|         var state = new ExportStateRecord( | ||||
|             Id: TrivyDbFeedExporter.ExporterId, | ||||
|             BaseExportId: "20240810T000000Z", | ||||
| @@ -56,20 +63,23 @@ public sealed class TrivyDbExportPlannerTests | ||||
|             ExportCursor: "sha256:old", | ||||
|             TargetRepository: "feedser/trivy", | ||||
|             ExporterVersion: "1.0", | ||||
|             UpdatedAt: DateTimeOffset.UtcNow); | ||||
|             UpdatedAt: DateTimeOffset.UtcNow, | ||||
|             Files: manifest); | ||||
|  | ||||
|         var plan = planner.CreatePlan(state, "sha256:new"); | ||||
|         var newManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:b") }; | ||||
|         var plan = planner.CreatePlan(state, "sha256:new", newManifest); | ||||
|  | ||||
|         Assert.Equal(TrivyDbExportMode.Full, plan.Mode); | ||||
|         Assert.Equal(TrivyDbExportMode.Delta, plan.Mode); | ||||
|         Assert.Equal("sha256:new", plan.TreeDigest); | ||||
|         Assert.Equal("20240810T000000Z", plan.BaseExportId); | ||||
|         Assert.Equal("sha256:base", plan.BaseManifestDigest); | ||||
|         Assert.False(plan.ResetBaseline); | ||||
|         Assert.Single(plan.ChangedFiles); | ||||
|  | ||||
|         var deltaState = state with { LastDeltaDigest = "sha256:delta" }; | ||||
|         var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer"); | ||||
|         var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer", newManifest); | ||||
|  | ||||
|         Assert.Equal(TrivyDbExportMode.Full, deltaPlan.Mode); | ||||
|         Assert.Equal(TrivyDbExportMode.Delta, deltaPlan.Mode); | ||||
|         Assert.True(deltaPlan.ResetBaseline); | ||||
|     } | ||||
| } | ||||
|   | ||||
| @@ -1,3 +1,4 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Linq; | ||||
| @@ -315,7 +316,8 @@ public sealed class TrivyDbFeedExporterTests : IDisposable | ||||
|             ExportCursor: "sha256:old", | ||||
|             TargetRepository: "registry.example/trivy", | ||||
|             ExporterVersion: "0.9.0", | ||||
|             UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30)); | ||||
|             UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30), | ||||
|             Files: Array.Empty<ExportFileRecord>()); | ||||
|         await stateStore.UpsertAsync(existingRecord, CancellationToken.None); | ||||
|  | ||||
|         var stateManager = new ExportStateManager(stateStore, timeProvider); | ||||
| @@ -350,6 +352,7 @@ public sealed class TrivyDbFeedExporterTests : IDisposable | ||||
|         Assert.Null(updated.LastDeltaDigest); | ||||
|         Assert.NotEqual("sha256:old", updated.ExportCursor); | ||||
|         Assert.Equal("registry.example/trivy", updated.TargetRepository); | ||||
|         Assert.NotEmpty(updated.Files); | ||||
|     } | ||||
|  | ||||
|     private static Advisory CreateSampleAdvisory( | ||||
|   | ||||
| @@ -10,4 +10,4 @@ | ||||
| |End-to-end tests with small dataset|QA|Exporters|DONE – added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.| | ||||
| |ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE – baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.| | ||||
| |Streamed package building to avoid large copies|BE-Export|Exporters|DONE – metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.| | ||||
| |Plan incremental/delta exports|BE-Export|Exporters|TODO – design reuse of existing blobs/layers when inputs unchanged instead of rewriting full trees each run.| | ||||
| |Plan incremental/delta exports|BE-Export|Exporters|DOING – export state now persists per-file manifests; planner detects changes/removed files and schedules delta vs full runs, groundwork laid for layer reuse.| | ||||
|   | ||||
| @@ -1,8 +1,14 @@ | ||||
| namespace StellaOps.Feedser.Exporter.TrivyDb; | ||||
|  | ||||
| using System.Collections.Generic; | ||||
| using StellaOps.Feedser.Storage.Mongo.Exporting; | ||||
|  | ||||
| public sealed record TrivyDbExportPlan( | ||||
|     TrivyDbExportMode Mode, | ||||
|     string TreeDigest, | ||||
|     string? BaseExportId, | ||||
|     string? BaseManifestDigest, | ||||
|     bool ResetBaseline); | ||||
|     bool ResetBaseline, | ||||
|     IReadOnlyList<ExportFileRecord> Manifest, | ||||
|     IReadOnlyList<ExportFileRecord> ChangedFiles, | ||||
|     IReadOnlyList<string> RemovedPaths); | ||||
|   | ||||
| @@ -3,40 +3,100 @@ using StellaOps.Feedser.Storage.Mongo.Exporting; | ||||
|  | ||||
| namespace StellaOps.Feedser.Exporter.TrivyDb; | ||||
|  | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using StellaOps.Feedser.Storage.Mongo.Exporting; | ||||
|  | ||||
| public sealed class TrivyDbExportPlanner | ||||
| { | ||||
|     public TrivyDbExportPlan CreatePlan(ExportStateRecord? existingState, string treeDigest) | ||||
|     public TrivyDbExportPlan CreatePlan( | ||||
|         ExportStateRecord? existingState, | ||||
|         string treeDigest, | ||||
|         IReadOnlyList<ExportFileRecord> manifest) | ||||
|     { | ||||
|         ArgumentException.ThrowIfNullOrEmpty(treeDigest); | ||||
|         manifest ??= Array.Empty<ExportFileRecord>(); | ||||
|  | ||||
|         if (existingState is null) | ||||
|         if (existingState is null || (existingState.Files?.Count ?? 0) == 0) | ||||
|         { | ||||
|             return new TrivyDbExportPlan( | ||||
|                 TrivyDbExportMode.Full, | ||||
|                 treeDigest, | ||||
|                 BaseExportId: null, | ||||
|                 BaseManifestDigest: null, | ||||
|                 ResetBaseline: true); | ||||
|                 BaseExportId: existingState?.BaseExportId, | ||||
|                 BaseManifestDigest: existingState?.LastFullDigest, | ||||
|                 ResetBaseline: true, | ||||
|                 Manifest: manifest, | ||||
|                 ChangedFiles: manifest, | ||||
|                 RemovedPaths: Array.Empty<string>()); | ||||
|         } | ||||
|  | ||||
|         if (string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal)) | ||||
|         var existingFiles = existingState.Files ?? Array.Empty<ExportFileRecord>(); | ||||
|         var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal); | ||||
|         if (cursorMatches) | ||||
|         { | ||||
|             return new TrivyDbExportPlan( | ||||
|                 TrivyDbExportMode.Skip, | ||||
|                 treeDigest, | ||||
|                 existingState.BaseExportId, | ||||
|                 existingState.LastFullDigest, | ||||
|                 ResetBaseline: false); | ||||
|                 ResetBaseline: false, | ||||
|                 Manifest: existingFiles, | ||||
|                 ChangedFiles: Array.Empty<ExportFileRecord>(), | ||||
|                 RemovedPaths: Array.Empty<string>()); | ||||
|         } | ||||
|  | ||||
|         var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); | ||||
|         var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         var removed = existingMap.Keys | ||||
|             .Where(path => !newMap.ContainsKey(path)) | ||||
|             .ToArray(); | ||||
|  | ||||
|         if (removed.Length > 0) | ||||
|         { | ||||
|             return new TrivyDbExportPlan( | ||||
|                 TrivyDbExportMode.Full, | ||||
|                 treeDigest, | ||||
|                 existingState.BaseExportId, | ||||
|                 existingState.LastFullDigest, | ||||
|                 ResetBaseline: true, | ||||
|                 Manifest: manifest, | ||||
|                 ChangedFiles: manifest, | ||||
|                 RemovedPaths: removed); | ||||
|         } | ||||
|  | ||||
|         var changed = new List<ExportFileRecord>(); | ||||
|         foreach (var file in manifest) | ||||
|         { | ||||
|             if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal)) | ||||
|             { | ||||
|                 changed.Add(file); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (changed.Count == 0) | ||||
|         { | ||||
|             return new TrivyDbExportPlan( | ||||
|                 TrivyDbExportMode.Skip, | ||||
|                 treeDigest, | ||||
|                 existingState.BaseExportId, | ||||
|                 existingState.LastFullDigest, | ||||
|                 ResetBaseline: false, | ||||
|                 Manifest: existingFiles, | ||||
|                 ChangedFiles: Array.Empty<ExportFileRecord>(), | ||||
|                 RemovedPaths: Array.Empty<string>()); | ||||
|         } | ||||
|  | ||||
|         var resetBaseline = existingState.LastDeltaDigest is not null; | ||||
|  | ||||
|         // Placeholder for future delta support – current behavior always rebuilds when tree changes. | ||||
|         return new TrivyDbExportPlan( | ||||
|             TrivyDbExportMode.Full, | ||||
|             TrivyDbExportMode.Delta, | ||||
|             treeDigest, | ||||
|             existingState.BaseExportId, | ||||
|             existingState.LastFullDigest, | ||||
|             resetBaseline); | ||||
|             resetBaseline, | ||||
|             Manifest: manifest, | ||||
|             ChangedFiles: changed, | ||||
|             RemovedPaths: Array.Empty<string>()); | ||||
|     } | ||||
| } | ||||
|   | ||||
| @@ -85,9 +85,13 @@ public sealed class TrivyDbFeedExporter : IFeedExporter | ||||
|             jsonResult.AdvisoryCount, | ||||
|             jsonResult.TotalBytes); | ||||
|  | ||||
|         var manifest = jsonResult.Files | ||||
|             .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest)) | ||||
|             .ToArray(); | ||||
|  | ||||
|         var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult); | ||||
|         var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); | ||||
|         var plan = _exportPlanner.CreatePlan(existingState, treeDigest); | ||||
|         var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest); | ||||
|  | ||||
|         if (plan.Mode == TrivyDbExportMode.Skip) | ||||
|         { | ||||
| @@ -104,6 +108,14 @@ public sealed class TrivyDbFeedExporter : IFeedExporter | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         if (plan.Mode == TrivyDbExportMode.Delta) | ||||
|         { | ||||
|             _logger.LogInformation( | ||||
|                 "Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.", | ||||
|                 exportId, | ||||
|                 plan.ChangedFiles.Count); | ||||
|         } | ||||
|  | ||||
|         var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false); | ||||
|         var metadataBytes = CreateMetadataJson(builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt); | ||||
|  | ||||
| @@ -142,15 +154,29 @@ public sealed class TrivyDbFeedExporter : IFeedExporter | ||||
|                 resetBaseline = true; | ||||
|             } | ||||
|  | ||||
|             await _stateManager.StoreFullExportAsync( | ||||
|                 ExporterId, | ||||
|                 exportId, | ||||
|                 ociResult.ManifestDigest, | ||||
|                 cursor: treeDigest, | ||||
|                 targetRepository: _options.TargetRepository, | ||||
|                 exporterVersion: _exporterVersion, | ||||
|                 resetBaseline: resetBaseline, | ||||
|                 cancellationToken: cancellationToken).ConfigureAwait(false); | ||||
|             if (plan.Mode == TrivyDbExportMode.Full || resetBaseline) | ||||
|             { | ||||
|                 await _stateManager.StoreFullExportAsync( | ||||
|                     ExporterId, | ||||
|                     exportId, | ||||
|                     ociResult.ManifestDigest, | ||||
|                     cursor: treeDigest, | ||||
|                     targetRepository: _options.TargetRepository, | ||||
|                     exporterVersion: _exporterVersion, | ||||
|                     resetBaseline: resetBaseline, | ||||
|                     manifest: plan.Manifest, | ||||
|                     cancellationToken: cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             else | ||||
|             { | ||||
|                 await _stateManager.StoreDeltaExportAsync( | ||||
|                     ExporterId, | ||||
|                     deltaDigest: treeDigest, | ||||
|                     cursor: treeDigest, | ||||
|                     exporterVersion: _exporterVersion, | ||||
|                     manifest: plan.Manifest, | ||||
|                     cancellationToken: cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|  | ||||
|             await CreateOfflineBundleAsync(destination, exportId, exportedAt, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|   | ||||
| @@ -16,6 +16,7 @@ public sealed class AdvisoryPrecedenceMergerTests | ||||
|     { | ||||
|         var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero)); | ||||
|         var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); | ||||
|         using var metrics = new MetricCollector("StellaOps.Feedser.Merge"); | ||||
|  | ||||
|         var (redHat, nvd) = CreateVendorAndRegistryAdvisories(); | ||||
|         var expectedMergeTimestamp = timeProvider.GetUtcNow(); | ||||
| @@ -46,6 +47,14 @@ public sealed class AdvisoryPrecedenceMergerTests | ||||
|         Assert.Equal(expectedMergeTimestamp, mergeProvenance.RecordedAt); | ||||
|         Assert.Contains("redhat", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); | ||||
|         Assert.Contains("nvd", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); | ||||
|  | ||||
|         var rangeMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.range_overrides"); | ||||
|         Assert.Equal(1, rangeMeasurement.Value); | ||||
|         Assert.Contains(rangeMeasurement.Tags, tag => string.Equals(tag.Key, "suppressed_source", StringComparison.Ordinal) && tag.Value?.ToString()?.Contains("nvd", StringComparison.OrdinalIgnoreCase) == true); | ||||
|  | ||||
|         var severityConflict = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.conflicts"); | ||||
|         Assert.Equal(1, severityConflict.Value); | ||||
|         Assert.Contains(severityConflict.Tags, tag => string.Equals(tag.Key, "type", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "severity", StringComparison.OrdinalIgnoreCase)); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
| @@ -155,6 +164,13 @@ public sealed class AdvisoryPrecedenceMergerTests | ||||
|         Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "primary_source" && string.Equals(tag.Value?.ToString(), "nvd", StringComparison.OrdinalIgnoreCase)); | ||||
|         Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "suppressed_source" && tag.Value?.ToString()?.Contains("redhat", StringComparison.OrdinalIgnoreCase) == true); | ||||
|  | ||||
|         Assert.DoesNotContain(metrics.Measurements, measurement => measurement.Name == "feedser.merge.range_overrides"); | ||||
|  | ||||
|         var conflictMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.conflicts"); | ||||
|         Assert.Equal(1, conflictMeasurement.Value); | ||||
|         Assert.Contains(conflictMeasurement.Tags, tag => tag.Key == "type" && string.Equals(tag.Value?.ToString(), "severity", StringComparison.OrdinalIgnoreCase)); | ||||
|         Assert.Contains(conflictMeasurement.Tags, tag => tag.Key == "reason" && string.Equals(tag.Value?.ToString(), "mismatch", StringComparison.OrdinalIgnoreCase)); | ||||
|  | ||||
|         var logEntry = Assert.Single(logger.Entries, entry => entry.EventId.Name == "AdvisoryOverride"); | ||||
|         Assert.Equal(LogLevel.Information, logEntry.Level); | ||||
|         Assert.NotNull(logEntry.StructuredState); | ||||
|   | ||||
| @@ -45,14 +45,21 @@ public sealed class AffectedPackagePrecedenceResolverTests | ||||
|             }); | ||||
|  | ||||
|         var resolver = new AffectedPackagePrecedenceResolver(); | ||||
|         var merged = resolver.Merge(new[] { nvd, redHat }); | ||||
|         var result = resolver.Merge(new[] { nvd, redHat }); | ||||
|  | ||||
|         var package = Assert.Single(merged); | ||||
|         var package = Assert.Single(result.Packages); | ||||
|         Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", package.Identifier); | ||||
|         Assert.Empty(package.VersionRanges); // NVD range overridden | ||||
|         Assert.Contains(package.Statuses, status => status.Status == "known_affected"); | ||||
|         Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); | ||||
|         Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); | ||||
|  | ||||
|         var rangeOverride = Assert.Single(result.Overrides); | ||||
|         Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", rangeOverride.Identifier); | ||||
|         Assert.Equal(0, rangeOverride.PrimaryRank); | ||||
|         Assert.True(rangeOverride.SuppressedRank >= rangeOverride.PrimaryRank); | ||||
|         Assert.Equal(0, rangeOverride.PrimaryRangeCount); | ||||
|         Assert.Equal(1, rangeOverride.SuppressedRangeCount); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
| @@ -78,11 +85,12 @@ public sealed class AffectedPackagePrecedenceResolverTests | ||||
|             }); | ||||
|  | ||||
|         var resolver = new AffectedPackagePrecedenceResolver(); | ||||
|         var merged = resolver.Merge(new[] { nvd }); | ||||
|         var result = resolver.Merge(new[] { nvd }); | ||||
|  | ||||
|         var package = Assert.Single(merged); | ||||
|         var package = Assert.Single(result.Packages); | ||||
|         Assert.Equal(nvd.Identifier, package.Identifier); | ||||
|         Assert.Equal(nvd.VersionRanges.Single().RangeExpression, package.VersionRanges.Single().RangeExpression); | ||||
|         Assert.Equal("nvd", package.Provenance.Single().Source); | ||||
|         Assert.Empty(result.Overrides); | ||||
|     } | ||||
| } | ||||
|   | ||||
							
								
								
									
										135
									
								
								src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										135
									
								
								src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,135 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging.Abstractions; | ||||
| using MongoDB.Driver; | ||||
| using StellaOps.Feedser.Merge.Services; | ||||
| using StellaOps.Feedser.Storage.Mongo; | ||||
| using StellaOps.Feedser.Storage.Mongo.Aliases; | ||||
| using StellaOps.Feedser.Testing; | ||||
|  | ||||
| namespace StellaOps.Feedser.Merge.Tests; | ||||
|  | ||||
| [Collection("mongo-fixture")] | ||||
| public sealed class AliasGraphResolverTests : IClassFixture<MongoIntegrationFixture> | ||||
| { | ||||
|     private readonly MongoIntegrationFixture _fixture; | ||||
|  | ||||
|     public AliasGraphResolverTests(MongoIntegrationFixture fixture) | ||||
|     { | ||||
|         _fixture = fixture; | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task ResolveAsync_ReturnsCollisions_WhenAliasesOverlap() | ||||
|     { | ||||
|         await DropAliasCollectionAsync(); | ||||
|  | ||||
|         var aliasStore = new AliasStore(_fixture.Database, NullLogger<AliasStore>.Instance); | ||||
|         var resolver = new AliasGraphResolver(aliasStore); | ||||
|  | ||||
|         var timestamp = DateTimeOffset.UtcNow; | ||||
|         await aliasStore.ReplaceAsync( | ||||
|             "ADV-1", | ||||
|             new[] { new AliasEntry("CVE", "CVE-2025-2000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-1") }, | ||||
|             timestamp, | ||||
|             CancellationToken.None); | ||||
|  | ||||
|         await aliasStore.ReplaceAsync( | ||||
|             "ADV-2", | ||||
|             new[] { new AliasEntry("CVE", "CVE-2025-2000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-2") }, | ||||
|             timestamp.AddMinutes(1), | ||||
|             CancellationToken.None); | ||||
|  | ||||
|         var result = await resolver.ResolveAsync("ADV-1", CancellationToken.None); | ||||
|         Assert.NotNull(result); | ||||
|         Assert.Equal("ADV-1", result.AdvisoryKey); | ||||
|         Assert.NotEmpty(result.Collisions); | ||||
|         var collision = Assert.Single(result.Collisions); | ||||
|         Assert.Equal("CVE", collision.Scheme); | ||||
|         Assert.Contains("ADV-1", collision.AdvisoryKeys); | ||||
|         Assert.Contains("ADV-2", collision.AdvisoryKeys); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task BuildComponentAsync_TracesConnectedAdvisories() | ||||
|     { | ||||
|         await DropAliasCollectionAsync(); | ||||
|         var aliasStore = new AliasStore(_fixture.Database, NullLogger<AliasStore>.Instance); | ||||
|         var resolver = new AliasGraphResolver(aliasStore); | ||||
|  | ||||
|         var timestamp = DateTimeOffset.UtcNow; | ||||
|         await aliasStore.ReplaceAsync( | ||||
|             "ADV-A", | ||||
|             new[] { new AliasEntry("CVE", "CVE-2025-4000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-A") }, | ||||
|             timestamp, | ||||
|             CancellationToken.None); | ||||
|  | ||||
|         await aliasStore.ReplaceAsync( | ||||
|             "ADV-B", | ||||
|             new[] { new AliasEntry("CVE", "CVE-2025-4000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-B"), new AliasEntry("OSV", "OSV-2025-1") }, | ||||
|             timestamp.AddMinutes(1), | ||||
|             CancellationToken.None); | ||||
|  | ||||
|         await aliasStore.ReplaceAsync( | ||||
|             "ADV-C", | ||||
|             new[] { new AliasEntry("OSV", "OSV-2025-1"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-C") }, | ||||
|             timestamp.AddMinutes(2), | ||||
|             CancellationToken.None); | ||||
|  | ||||
|         var component = await resolver.BuildComponentAsync("ADV-A", CancellationToken.None); | ||||
|         Assert.Contains("ADV-A", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); | ||||
|         Assert.Contains("ADV-B", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); | ||||
|         Assert.Contains("ADV-C", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); | ||||
|         Assert.NotEmpty(component.Collisions); | ||||
|         Assert.True(component.AliasMap.ContainsKey("ADV-A")); | ||||
|         Assert.Contains(component.AliasMap["ADV-B"], record => record.Scheme == "OSV" && record.Value == "OSV-2025-1"); | ||||
|     } | ||||
|  | ||||
|     private async Task DropAliasCollectionAsync() | ||||
|     { | ||||
|         try | ||||
|         { | ||||
|             await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.Alias); | ||||
|         } | ||||
|         catch (MongoDB.Driver.MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task BuildComponentAsync_LinksOsvAndGhsaAliases() | ||||
|     { | ||||
|         await DropAliasCollectionAsync(); | ||||
|  | ||||
|         var aliasStore = new AliasStore(_fixture.Database, NullLogger<AliasStore>.Instance); | ||||
|         var resolver = new AliasGraphResolver(aliasStore); | ||||
|         var timestamp = DateTimeOffset.UtcNow; | ||||
|  | ||||
|         await aliasStore.ReplaceAsync( | ||||
|             "ADV-OSV", | ||||
|             new[] | ||||
|             { | ||||
|                 new AliasEntry("OSV", "OSV-2025-2001"), | ||||
|                 new AliasEntry("GHSA", "GHSA-zzzz-zzzz-zzzz"), | ||||
|                 new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-OSV"), | ||||
|             }, | ||||
|             timestamp, | ||||
|             CancellationToken.None); | ||||
|  | ||||
|         await aliasStore.ReplaceAsync( | ||||
|             "ADV-GHSA", | ||||
|             new[] | ||||
|             { | ||||
|                 new AliasEntry("GHSA", "GHSA-zzzz-zzzz-zzzz"), | ||||
|                 new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-GHSA"), | ||||
|             }, | ||||
|             timestamp.AddMinutes(1), | ||||
|             CancellationToken.None); | ||||
|  | ||||
|         var component = await resolver.BuildComponentAsync("ADV-OSV", CancellationToken.None); | ||||
|  | ||||
|         Assert.Contains("ADV-GHSA", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); | ||||
|         Assert.Contains(component.Collisions, collision => collision.Scheme == "GHSA" && collision.Value == "GHSA-zzzz-zzzz-zzzz"); | ||||
|     } | ||||
| } | ||||
| @@ -76,6 +76,26 @@ public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime | ||||
|         Assert.True(persisted.BeforeHash.Length > 0); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task MergePipeline_IsDeterministicAcrossRuns() | ||||
|     { | ||||
|         await EnsureInitializedAsync(); | ||||
|  | ||||
|         var merger = _merger!; | ||||
|         var calculator = new CanonicalHashCalculator(); | ||||
|  | ||||
|         var first = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); | ||||
|         var second = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); | ||||
|  | ||||
|         var firstHash = calculator.ComputeHash(first); | ||||
|         var secondHash = calculator.ComputeHash(second); | ||||
|  | ||||
|         Assert.Equal(firstHash, secondHash); | ||||
|         Assert.Equal(first.AdvisoryKey, second.AdvisoryKey); | ||||
|         Assert.Equal(first.Aliases.Length, second.Aliases.Length); | ||||
|         Assert.True(first.Aliases.SequenceEqual(second.Aliases)); | ||||
|     } | ||||
|  | ||||
|     public async Task InitializeAsync() | ||||
|     { | ||||
|         _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero)) | ||||
|   | ||||
							
								
								
									
										6
									
								
								src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										6
									
								
								src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,6 @@ | ||||
| namespace StellaOps.Feedser.Merge.Jobs; | ||||
|  | ||||
| internal static class MergeJobKinds | ||||
| { | ||||
|     public const string Reconcile = "merge:reconcile"; | ||||
| } | ||||
							
								
								
									
										43
									
								
								src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										43
									
								
								src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,43 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
| using StellaOps.Feedser.Merge.Services; | ||||
|  | ||||
| namespace StellaOps.Feedser.Merge.Jobs; | ||||
|  | ||||
| public sealed class MergeReconcileJob : IJob | ||||
| { | ||||
|     private readonly AdvisoryMergeService _mergeService; | ||||
|     private readonly ILogger<MergeReconcileJob> _logger; | ||||
|  | ||||
|     public MergeReconcileJob(AdvisoryMergeService mergeService, ILogger<MergeReconcileJob> logger) | ||||
|     { | ||||
|         _mergeService = mergeService ?? throw new ArgumentNullException(nameof(mergeService)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|     { | ||||
|         if (!context.Parameters.TryGetValue("seed", out var seedValue) || seedValue is not string seed || string.IsNullOrWhiteSpace(seed)) | ||||
|         { | ||||
|             context.Logger.LogWarning("merge:reconcile job requires a non-empty 'seed' parameter."); | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var result = await _mergeService.MergeAsync(seed, cancellationToken).ConfigureAwait(false); | ||||
|         if (result.Merged is null) | ||||
|         { | ||||
|             _logger.LogInformation("No advisories available to merge for alias component seeded by {Seed}", seed); | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         _logger.LogInformation( | ||||
|             "Merged alias component seeded by {Seed} into canonical {Canonical} using {Count} advisories; collisions={Collisions}", | ||||
|             seed, | ||||
|             result.CanonicalAdvisoryKey, | ||||
|             result.Inputs.Count, | ||||
|             result.Component.Collisions.Count); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,41 @@ | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.DependencyInjection.Extensions; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Feedser.Merge.Jobs; | ||||
| using StellaOps.Feedser.Merge.Options; | ||||
| using StellaOps.Feedser.Merge.Services; | ||||
|  | ||||
| namespace StellaOps.Feedser.Merge; | ||||
|  | ||||
| public static class MergeServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddMergeModule(this IServiceCollection services, IConfiguration configuration) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         services.TryAddSingleton<CanonicalHashCalculator>(); | ||||
|         services.TryAddSingleton<AliasGraphResolver>(); | ||||
|         services.TryAddSingleton<AffectedPackagePrecedenceResolver>(sp => | ||||
|         { | ||||
|             var options = configuration.GetSection("feedser:merge:precedence").Get<AdvisoryPrecedenceOptions>(); | ||||
|             return options is null ? new AffectedPackagePrecedenceResolver() : new AffectedPackagePrecedenceResolver(options); | ||||
|         }); | ||||
|  | ||||
|         services.TryAddSingleton<AdvisoryPrecedenceMerger>(sp => | ||||
|         { | ||||
|             var resolver = sp.GetRequiredService<AffectedPackagePrecedenceResolver>(); | ||||
|             var options = configuration.GetSection("feedser:merge:precedence").Get<AdvisoryPrecedenceOptions>(); | ||||
|             var timeProvider = sp.GetRequiredService<TimeProvider>(); | ||||
|             var logger = sp.GetRequiredService<ILogger<AdvisoryPrecedenceMerger>>(); | ||||
|             return new AdvisoryPrecedenceMerger(resolver, options, timeProvider, logger); | ||||
|         }); | ||||
|  | ||||
|         services.TryAddSingleton<MergeEventWriter>(); | ||||
|         services.TryAddSingleton<AdvisoryMergeService>(); | ||||
|         services.AddTransient<MergeReconcileJob>(); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,96 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Merge.Options; | ||||
|  | ||||
| /// <summary> | ||||
| /// Provides the built-in precedence table used by the merge engine when no overrides are supplied. | ||||
| /// </summary> | ||||
| internal static class AdvisoryPrecedenceDefaults | ||||
| { | ||||
|     public static IReadOnlyDictionary<string, int> Rankings { get; } = CreateDefaultTable(); | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, int> CreateDefaultTable() | ||||
|     { | ||||
|         var table = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         // 0 – distro PSIRTs/OVAL feeds (authoritative for OS packages). | ||||
|         Add(table, 0, | ||||
|             "redhat", | ||||
|             "ubuntu", | ||||
|             "distro-ubuntu", | ||||
|             "debian", | ||||
|             "distro-debian", | ||||
|             "suse", | ||||
|             "distro-suse"); | ||||
|  | ||||
|         // 1 – vendor PSIRTs (authoritative product advisories). | ||||
|         Add(table, 1, | ||||
|             "msrc", | ||||
|             "vndr-msrc", | ||||
|             "vndr-oracle", | ||||
|             "vndr_oracle", | ||||
|             "oracle", | ||||
|             "vndr-adobe", | ||||
|             "adobe", | ||||
|             "vndr-apple", | ||||
|             "apple", | ||||
|             "vndr-cisco", | ||||
|             "cisco", | ||||
|             "vmware", | ||||
|             "vndr-vmware", | ||||
|             "vndr_vmware", | ||||
|             "vndr-chromium", | ||||
|             "chromium", | ||||
|             "vendor"); | ||||
|  | ||||
|         // 2 – ecosystem registries (OSS package maintainers). | ||||
|         Add(table, 2, | ||||
|             "ghsa", | ||||
|             "osv", | ||||
|             "cve"); | ||||
|  | ||||
|         // 3 – regional CERT / ICS enrichment feeds. | ||||
|         Add(table, 3, | ||||
|             "jvn", | ||||
|             "acsc", | ||||
|             "cccs", | ||||
|             "cert-fr", | ||||
|             "certfr", | ||||
|             "cert-in", | ||||
|             "certin", | ||||
|             "cert-cc", | ||||
|             "certcc", | ||||
|             "certbund", | ||||
|             "cert-bund", | ||||
|             "ru-bdu", | ||||
|             "ru-nkcki", | ||||
|             "kisa", | ||||
|             "ics-cisa", | ||||
|             "ics-kaspersky"); | ||||
|  | ||||
|         // 4 – KEV / exploit catalogue annotations (flag only). | ||||
|         Add(table, 4, | ||||
|             "kev", | ||||
|             "cisa-kev"); | ||||
|  | ||||
|         // 5 – public registries (baseline data). | ||||
|         Add(table, 5, | ||||
|             "nvd"); | ||||
|  | ||||
|         return table; | ||||
|     } | ||||
|  | ||||
|     private static void Add(IDictionary<string, int> table, int rank, params string[] sources) | ||||
|     { | ||||
|         foreach (var source in sources) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(source)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             table[source] = rank; | ||||
|         } | ||||
|     } | ||||
| } | ||||
							
								
								
									
										190
									
								
								src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										190
									
								
								src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,190 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Diagnostics.Metrics; | ||||
| using System.Linq; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Storage.Mongo.Advisories; | ||||
| using StellaOps.Feedser.Storage.Mongo.Aliases; | ||||
| using StellaOps.Feedser.Storage.Mongo.MergeEvents; | ||||
|  | ||||
| namespace StellaOps.Feedser.Merge.Services; | ||||
|  | ||||
| public sealed class AdvisoryMergeService | ||||
| { | ||||
|     private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); | ||||
|     private static readonly Counter<long> AliasCollisionCounter = MergeMeter.CreateCounter<long>( | ||||
|         "feedser.merge.identity_conflicts", | ||||
|         unit: "count", | ||||
|         description: "Number of alias collisions detected during merge."); | ||||
|  | ||||
|     private static readonly string[] PreferredAliasSchemes = | ||||
|     { | ||||
|         AliasSchemes.Cve, | ||||
|         AliasSchemes.Ghsa, | ||||
|         AliasSchemes.OsV, | ||||
|         AliasSchemes.Msrc, | ||||
|     }; | ||||
|  | ||||
|     private readonly AliasGraphResolver _aliasResolver; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly AdvisoryPrecedenceMerger _precedenceMerger; | ||||
|     private readonly MergeEventWriter _mergeEventWriter; | ||||
|     private readonly ILogger<AdvisoryMergeService> _logger; | ||||
|  | ||||
|     public AdvisoryMergeService( | ||||
|         AliasGraphResolver aliasResolver, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         AdvisoryPrecedenceMerger precedenceMerger, | ||||
|         MergeEventWriter mergeEventWriter, | ||||
|         ILogger<AdvisoryMergeService> logger) | ||||
|     { | ||||
|         _aliasResolver = aliasResolver ?? throw new ArgumentNullException(nameof(aliasResolver)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _precedenceMerger = precedenceMerger ?? throw new ArgumentNullException(nameof(precedenceMerger)); | ||||
|         _mergeEventWriter = mergeEventWriter ?? throw new ArgumentNullException(nameof(mergeEventWriter)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async Task<AdvisoryMergeResult> MergeAsync(string seedAdvisoryKey, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentException.ThrowIfNullOrWhiteSpace(seedAdvisoryKey); | ||||
|  | ||||
|         var component = await _aliasResolver.BuildComponentAsync(seedAdvisoryKey, cancellationToken).ConfigureAwait(false); | ||||
|         var inputs = new List<Advisory>(); | ||||
|  | ||||
|         foreach (var advisoryKey in component.AdvisoryKeys) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|             var advisory = await _advisoryStore.FindAsync(advisoryKey, cancellationToken).ConfigureAwait(false); | ||||
|             if (advisory is not null) | ||||
|             { | ||||
|                 inputs.Add(advisory); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (inputs.Count == 0) | ||||
|         { | ||||
|             _logger.LogWarning("Alias component seeded by {Seed} contains no persisted advisories", seedAdvisoryKey); | ||||
|             return AdvisoryMergeResult.Empty(seedAdvisoryKey, component); | ||||
|         } | ||||
|  | ||||
|         var canonicalKey = SelectCanonicalKey(component) ?? seedAdvisoryKey; | ||||
|         var before = await _advisoryStore.FindAsync(canonicalKey, cancellationToken).ConfigureAwait(false); | ||||
|         var normalizedInputs = NormalizeInputs(inputs, canonicalKey); | ||||
|  | ||||
|         Advisory? merged; | ||||
|         try | ||||
|         { | ||||
|             merged = _precedenceMerger.Merge(normalizedInputs); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _logger.LogError(ex, "Failed to merge alias component seeded by {Seed}", seedAdvisoryKey); | ||||
|             throw; | ||||
|         } | ||||
|  | ||||
|         if (component.Collisions.Count > 0) | ||||
|         { | ||||
|             foreach (var collision in component.Collisions) | ||||
|             { | ||||
|                 var tags = new KeyValuePair<string, object?>[] | ||||
|                 { | ||||
|                     new("scheme", collision.Scheme ?? string.Empty), | ||||
|                     new("alias_value", collision.Value ?? string.Empty), | ||||
|                     new("advisory_count", collision.AdvisoryKeys.Count), | ||||
|                 }; | ||||
|  | ||||
|                 AliasCollisionCounter.Add(1, tags); | ||||
|  | ||||
|                 _logger.LogInformation( | ||||
|                     "Alias collision {Scheme}:{Value} involves advisories {Advisories}", | ||||
|                     collision.Scheme, | ||||
|                     collision.Value, | ||||
|                     string.Join(", ", collision.AdvisoryKeys)); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (merged is not null) | ||||
|         { | ||||
|             await _advisoryStore.UpsertAsync(merged, cancellationToken).ConfigureAwait(false); | ||||
|             await _mergeEventWriter.AppendAsync( | ||||
|                 canonicalKey, | ||||
|                 before, | ||||
|                 merged, | ||||
|                 Array.Empty<Guid>(), | ||||
|                 cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|  | ||||
|         return new AdvisoryMergeResult(seedAdvisoryKey, canonicalKey, component, inputs, before, merged); | ||||
|     } | ||||
|  | ||||
|     private static IEnumerable<Advisory> NormalizeInputs(IEnumerable<Advisory> advisories, string canonicalKey) | ||||
|     { | ||||
|         foreach (var advisory in advisories) | ||||
|         { | ||||
|             yield return CloneWithKey(advisory, canonicalKey); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static Advisory CloneWithKey(Advisory source, string advisoryKey) | ||||
|         => new( | ||||
|             advisoryKey, | ||||
|             source.Title, | ||||
|             source.Summary, | ||||
|             source.Language, | ||||
|             source.Published, | ||||
|             source.Modified, | ||||
|             source.Severity, | ||||
|             source.ExploitKnown, | ||||
|             source.Aliases, | ||||
|             source.References, | ||||
|             source.AffectedPackages, | ||||
|             source.CvssMetrics, | ||||
|             source.Provenance); | ||||
|  | ||||
|     private static string? SelectCanonicalKey(AliasComponent component) | ||||
|     { | ||||
|         foreach (var scheme in PreferredAliasSchemes) | ||||
|         { | ||||
|             var alias = component.AliasMap.Values | ||||
|                 .SelectMany(static aliases => aliases) | ||||
|                 .FirstOrDefault(record => string.Equals(record.Scheme, scheme, StringComparison.OrdinalIgnoreCase)); | ||||
|             if (!string.IsNullOrWhiteSpace(alias?.Value)) | ||||
|             { | ||||
|                 return alias.Value; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (component.AliasMap.TryGetValue(component.SeedAdvisoryKey, out var seedAliases)) | ||||
|         { | ||||
|             var primary = seedAliases.FirstOrDefault(record => string.Equals(record.Scheme, AliasStoreConstants.PrimaryScheme, StringComparison.OrdinalIgnoreCase)); | ||||
|             if (!string.IsNullOrWhiteSpace(primary?.Value)) | ||||
|             { | ||||
|                 return primary.Value; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var firstAlias = component.AliasMap.Values.SelectMany(static aliases => aliases).FirstOrDefault(); | ||||
|         if (!string.IsNullOrWhiteSpace(firstAlias?.Value)) | ||||
|         { | ||||
|             return firstAlias.Value; | ||||
|         } | ||||
|  | ||||
|         return component.SeedAdvisoryKey; | ||||
|     } | ||||
| } | ||||
|  | ||||
| public sealed record AdvisoryMergeResult( | ||||
|     string SeedAdvisoryKey, | ||||
|     string CanonicalAdvisoryKey, | ||||
|     AliasComponent Component, | ||||
|     IReadOnlyList<Advisory> Inputs, | ||||
|     Advisory? Previous, | ||||
|     Advisory? Merged) | ||||
| { | ||||
|     public static AdvisoryMergeResult Empty(string seed, AliasComponent component) | ||||
|         => new(seed, seed, component, Array.Empty<Advisory>(), null, null); | ||||
| } | ||||
| @@ -1,6 +1,7 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Diagnostics.Metrics; | ||||
| using System.Globalization; | ||||
| using System.Linq; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Logging.Abstractions; | ||||
| @@ -14,36 +15,42 @@ namespace StellaOps.Feedser.Merge.Services; | ||||
| /// </summary> | ||||
| public sealed class AdvisoryPrecedenceMerger | ||||
| { | ||||
|     private static readonly IReadOnlyDictionary<string, int> DefaultPrecedence = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase) | ||||
|     { | ||||
|         ["redhat"] = 0, | ||||
|         ["ubuntu"] = 0, | ||||
|         ["debian"] = 0, | ||||
|         ["suse"] = 0, | ||||
|         ["msrc"] = 1, | ||||
|         ["oracle"] = 1, | ||||
|         ["adobe"] = 1, | ||||
|         ["chromium"] = 1, | ||||
|         ["vendor"] = 1, | ||||
|         ["jvn"] = 2, | ||||
|         ["certfr"] = 2, | ||||
|         ["certin"] = 2, | ||||
|         ["ics-kaspersky"] = 2, | ||||
|         ["kev"] = 6, | ||||
|         ["nvd"] = 5, | ||||
|     }; | ||||
|  | ||||
|     private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); | ||||
|     private static readonly Counter<long> MergeCounter = MergeMeter.CreateCounter<long>( | ||||
|         "feedser.merge.operations", | ||||
|         unit: "count", | ||||
|         description: "Number of merge invocations executed by the precedence engine."); | ||||
|  | ||||
|     private static readonly Counter<long> OverridesCounter = MergeMeter.CreateCounter<long>( | ||||
|         "feedser.merge.overrides", | ||||
|         unit: "count", | ||||
|         description: "Number of times lower-precedence advisories were overridden by higher-precedence sources."); | ||||
|  | ||||
|     private static readonly Counter<long> RangeOverrideCounter = MergeMeter.CreateCounter<long>( | ||||
|         "feedser.merge.range_overrides", | ||||
|         unit: "count", | ||||
|         description: "Number of affected-package range overrides performed during precedence merge."); | ||||
|  | ||||
|     private static readonly Counter<long> ConflictCounter = MergeMeter.CreateCounter<long>( | ||||
|         "feedser.merge.conflicts", | ||||
|         unit: "count", | ||||
|         description: "Number of precedence conflicts detected (severity, rank ties, etc.)."); | ||||
|  | ||||
|     private static readonly Action<ILogger, MergeOverrideAudit, Exception?> OverrideLogged = LoggerMessage.Define<MergeOverrideAudit>( | ||||
|         LogLevel.Information, | ||||
|         new EventId(1000, "AdvisoryOverride"), | ||||
|         "Advisory precedence override {@Override}"); | ||||
|  | ||||
|     private static readonly Action<ILogger, PackageOverrideAudit, Exception?> RangeOverrideLogged = LoggerMessage.Define<PackageOverrideAudit>( | ||||
|         LogLevel.Information, | ||||
|         new EventId(1001, "PackageRangeOverride"), | ||||
|         "Affected package precedence override {@Override}"); | ||||
|  | ||||
|     private static readonly Action<ILogger, MergeFieldConflictAudit, Exception?> ConflictLogged = LoggerMessage.Define<MergeFieldConflictAudit>( | ||||
|         LogLevel.Information, | ||||
|         new EventId(1002, "PrecedenceConflict"), | ||||
|         "Precedence conflict {@Conflict}"); | ||||
|  | ||||
|     private readonly AffectedPackagePrecedenceResolver _packageResolver; | ||||
|     private readonly IReadOnlyDictionary<string, int> _precedence; | ||||
|     private readonly int _fallbackRank; | ||||
| @@ -51,12 +58,12 @@ public sealed class AdvisoryPrecedenceMerger | ||||
|     private readonly ILogger<AdvisoryPrecedenceMerger> _logger; | ||||
|  | ||||
|     public AdvisoryPrecedenceMerger() | ||||
|         : this(new AffectedPackagePrecedenceResolver(), DefaultPrecedence, System.TimeProvider.System, NullLogger<AdvisoryPrecedenceMerger>.Instance) | ||||
|         : this(new AffectedPackagePrecedenceResolver(), TimeProvider.System) | ||||
|     { | ||||
|     } | ||||
|  | ||||
|     public AdvisoryPrecedenceMerger(AffectedPackagePrecedenceResolver packageResolver, System.TimeProvider? timeProvider = null) | ||||
|         : this(packageResolver, DefaultPrecedence, timeProvider ?? System.TimeProvider.System, NullLogger<AdvisoryPrecedenceMerger>.Instance) | ||||
|         : this(packageResolver, packageResolver?.Precedence ?? AdvisoryPrecedenceDefaults.Rankings, timeProvider ?? TimeProvider.System, NullLogger<AdvisoryPrecedenceMerger>.Instance) | ||||
|     { | ||||
|     } | ||||
|  | ||||
| @@ -119,6 +126,8 @@ public sealed class AdvisoryPrecedenceMerger | ||||
|             .ThenByDescending(entry => entry.Advisory.Provenance.Length) | ||||
|             .ToArray(); | ||||
|  | ||||
|         MergeCounter.Add(1, new KeyValuePair<string, object?>("inputs", list.Count)); | ||||
|  | ||||
|         var primary = ordered[0].Advisory; | ||||
|  | ||||
|         var title = PickString(ordered, advisory => advisory.Title) ?? advisoryKey; | ||||
| @@ -137,7 +146,8 @@ public sealed class AdvisoryPrecedenceMerger | ||||
|             .Distinct() | ||||
|             .ToArray(); | ||||
|  | ||||
|         var affectedPackages = _packageResolver.Merge(ordered.SelectMany(entry => entry.Advisory.AffectedPackages)); | ||||
|         var packageResult = _packageResolver.Merge(ordered.SelectMany(entry => entry.Advisory.AffectedPackages)); | ||||
|         var affectedPackages = packageResult.Packages; | ||||
|         var cvssMetrics = ordered | ||||
|             .SelectMany(entry => entry.Advisory.CvssMetrics) | ||||
|             .Distinct() | ||||
| @@ -168,6 +178,8 @@ public sealed class AdvisoryPrecedenceMerger | ||||
|         var exploitKnown = ordered.Any(entry => entry.Advisory.ExploitKnown); | ||||
|  | ||||
|         LogOverrides(advisoryKey, ordered); | ||||
|         LogPackageOverrides(advisoryKey, packageResult.Overrides); | ||||
|         RecordFieldConflicts(advisoryKey, ordered); | ||||
|  | ||||
|         return new Advisory( | ||||
|             advisoryKey, | ||||
| @@ -275,6 +287,125 @@ public sealed class AdvisoryPrecedenceMerger | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private void LogPackageOverrides(string advisoryKey, IReadOnlyList<AffectedPackageOverride> overrides) | ||||
|     { | ||||
|         if (overrides.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         foreach (var record in overrides) | ||||
|         { | ||||
|             var tags = new KeyValuePair<string, object?>[] | ||||
|             { | ||||
|                 new("advisory_key", advisoryKey), | ||||
|                 new("package_type", record.Type), | ||||
|                 new("primary_source", FormatSourceLabel(record.PrimarySources)), | ||||
|                 new("suppressed_source", FormatSourceLabel(record.SuppressedSources)), | ||||
|                 new("primary_rank", record.PrimaryRank), | ||||
|                 new("suppressed_rank", record.SuppressedRank), | ||||
|                 new("primary_range_count", record.PrimaryRangeCount), | ||||
|                 new("suppressed_range_count", record.SuppressedRangeCount), | ||||
|             }; | ||||
|  | ||||
|             RangeOverrideCounter.Add(1, tags); | ||||
|  | ||||
|             var audit = new PackageOverrideAudit( | ||||
|                 advisoryKey, | ||||
|                 record.Type, | ||||
|                 record.Identifier, | ||||
|                 record.Platform, | ||||
|                 record.PrimaryRank, | ||||
|                 record.SuppressedRank, | ||||
|                 record.PrimarySources, | ||||
|                 record.SuppressedSources, | ||||
|                 record.PrimaryRangeCount, | ||||
|                 record.SuppressedRangeCount); | ||||
|  | ||||
|             RangeOverrideLogged(_logger, audit, null); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private void RecordFieldConflicts(string advisoryKey, IReadOnlyList<AdvisoryEntry> ordered) | ||||
|     { | ||||
|         if (ordered.Count <= 1) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var primary = ordered[0]; | ||||
|         var primarySeverity = NormalizeSeverity(primary.Advisory.Severity); | ||||
|  | ||||
|         for (var i = 1; i < ordered.Count; i++) | ||||
|         { | ||||
|             var candidate = ordered[i]; | ||||
|             var candidateSeverity = NormalizeSeverity(candidate.Advisory.Severity); | ||||
|  | ||||
|             if (!string.IsNullOrEmpty(candidateSeverity)) | ||||
|             { | ||||
|                 var reason = string.IsNullOrEmpty(primarySeverity) ? "primary_missing" : "mismatch"; | ||||
|                 if (string.IsNullOrEmpty(primarySeverity) || !string.Equals(primarySeverity, candidateSeverity, StringComparison.OrdinalIgnoreCase)) | ||||
|                 { | ||||
|                     RecordConflict( | ||||
|                         advisoryKey, | ||||
|                         "severity", | ||||
|                         reason, | ||||
|                         primary, | ||||
|                         candidate, | ||||
|                         primarySeverity ?? "(none)", | ||||
|                         candidateSeverity); | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             if (candidate.Rank == primary.Rank) | ||||
|             { | ||||
|                 RecordConflict( | ||||
|                     advisoryKey, | ||||
|                     "precedence_tie", | ||||
|                     "equal_rank", | ||||
|                     primary, | ||||
|                     candidate, | ||||
|                     primary.Rank.ToString(CultureInfo.InvariantCulture), | ||||
|                     candidate.Rank.ToString(CultureInfo.InvariantCulture)); | ||||
|             } | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private void RecordConflict( | ||||
|         string advisoryKey, | ||||
|         string conflictType, | ||||
|         string reason, | ||||
|         AdvisoryEntry primary, | ||||
|         AdvisoryEntry suppressed, | ||||
|         string? primaryValue, | ||||
|         string? suppressedValue) | ||||
|     { | ||||
|         var tags = new KeyValuePair<string, object?>[] | ||||
|         { | ||||
|             new("type", conflictType), | ||||
|             new("reason", reason), | ||||
|             new("primary_source", FormatSourceLabel(primary.Sources)), | ||||
|             new("suppressed_source", FormatSourceLabel(suppressed.Sources)), | ||||
|             new("primary_rank", primary.Rank), | ||||
|             new("suppressed_rank", suppressed.Rank), | ||||
|         }; | ||||
|  | ||||
|         ConflictCounter.Add(1, tags); | ||||
|  | ||||
|         var audit = new MergeFieldConflictAudit( | ||||
|             advisoryKey, | ||||
|             conflictType, | ||||
|             reason, | ||||
|             primary.Sources, | ||||
|             primary.Rank, | ||||
|             suppressed.Sources, | ||||
|             suppressed.Rank, | ||||
|             primaryValue, | ||||
|             suppressedValue); | ||||
|  | ||||
|         ConflictLogged(_logger, audit, null); | ||||
|     } | ||||
|  | ||||
|     private readonly record struct AdvisoryEntry(Advisory Advisory, int Rank) | ||||
|     { | ||||
|         public IReadOnlyCollection<string> Sources { get; } = Advisory.Provenance | ||||
| @@ -284,12 +415,15 @@ public sealed class AdvisoryPrecedenceMerger | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static string? NormalizeSeverity(string? severity) | ||||
|         => SeverityNormalization.Normalize(severity); | ||||
|  | ||||
|     private static AffectedPackagePrecedenceResolver EnsureResolver( | ||||
|         AffectedPackagePrecedenceResolver? resolver, | ||||
|         AdvisoryPrecedenceOptions? options, | ||||
|         out IReadOnlyDictionary<string, int> precedence) | ||||
|     { | ||||
|         precedence = AdvisoryPrecedenceTable.Merge(DefaultPrecedence, options); | ||||
|         precedence = AdvisoryPrecedenceTable.Merge(AdvisoryPrecedenceDefaults.Rankings, options); | ||||
|  | ||||
|         if (resolver is null) | ||||
|         { | ||||
| @@ -354,4 +488,27 @@ public sealed class AdvisoryPrecedenceMerger | ||||
|         int SuppressedAliasCount, | ||||
|         int PrimaryProvenanceCount, | ||||
|         int SuppressedProvenanceCount); | ||||
|  | ||||
|     private readonly record struct PackageOverrideAudit( | ||||
|         string AdvisoryKey, | ||||
|         string PackageType, | ||||
|         string Identifier, | ||||
|         string? Platform, | ||||
|         int PrimaryRank, | ||||
|         int SuppressedRank, | ||||
|         IReadOnlyCollection<string> PrimarySources, | ||||
|         IReadOnlyCollection<string> SuppressedSources, | ||||
|         int PrimaryRangeCount, | ||||
|         int SuppressedRangeCount); | ||||
|  | ||||
|     private readonly record struct MergeFieldConflictAudit( | ||||
|         string AdvisoryKey, | ||||
|         string ConflictType, | ||||
|         string Reason, | ||||
|         IReadOnlyCollection<string> PrimarySources, | ||||
|         int PrimaryRank, | ||||
|         IReadOnlyCollection<string> SuppressedSources, | ||||
|         int SuppressedRank, | ||||
|         string? PrimaryValue, | ||||
|         string? SuppressedValue); | ||||
| } | ||||
|   | ||||
| @@ -12,30 +12,16 @@ namespace StellaOps.Feedser.Merge.Services; | ||||
| /// </summary> | ||||
| public sealed class AffectedPackagePrecedenceResolver | ||||
| { | ||||
|     private static readonly IReadOnlyDictionary<string, int> DefaultPrecedence = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase) | ||||
|     { | ||||
|         ["redhat"] = 0, | ||||
|         ["ubuntu"] = 0, | ||||
|         ["debian"] = 0, | ||||
|         ["suse"] = 0, | ||||
|         ["msrc"] = 1, | ||||
|         ["oracle"] = 1, | ||||
|         ["adobe"] = 1, | ||||
|         ["chromium"] = 1, | ||||
|         ["vendor"] = 1, | ||||
|         ["nvd"] = 5, | ||||
|     }; | ||||
|  | ||||
|     private readonly IReadOnlyDictionary<string, int> _precedence; | ||||
|     private readonly int _fallbackRank; | ||||
|  | ||||
|     public AffectedPackagePrecedenceResolver() | ||||
|         : this(DefaultPrecedence) | ||||
|         : this(AdvisoryPrecedenceDefaults.Rankings) | ||||
|     { | ||||
|     } | ||||
|  | ||||
|     public AffectedPackagePrecedenceResolver(AdvisoryPrecedenceOptions? options) | ||||
|         : this(AdvisoryPrecedenceTable.Merge(DefaultPrecedence, options)) | ||||
|         : this(AdvisoryPrecedenceTable.Merge(AdvisoryPrecedenceDefaults.Rankings, options)) | ||||
|     { | ||||
|     } | ||||
|  | ||||
| @@ -47,7 +33,7 @@ public sealed class AffectedPackagePrecedenceResolver | ||||
|  | ||||
|     public IReadOnlyDictionary<string, int> Precedence => _precedence; | ||||
|  | ||||
|     public IReadOnlyList<AffectedPackage> Merge(IEnumerable<AffectedPackage> packages) | ||||
|     public AffectedPackagePrecedenceResult Merge(IEnumerable<AffectedPackage> packages) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(packages); | ||||
|  | ||||
| @@ -56,41 +42,66 @@ public sealed class AffectedPackagePrecedenceResolver | ||||
|             .GroupBy(pkg => (pkg.Type, pkg.Identifier, pkg.Platform ?? string.Empty)); | ||||
|  | ||||
|         var resolved = new List<AffectedPackage>(); | ||||
|         var overrides = new List<AffectedPackageOverride>(); | ||||
|  | ||||
|         foreach (var group in grouped) | ||||
|         { | ||||
|             var ordered = group | ||||
|                 .OrderBy(GetPrecedence) | ||||
|                 .ThenByDescending(static pkg => pkg.Provenance.Length) | ||||
|                 .ThenByDescending(static pkg => pkg.VersionRanges.Length); | ||||
|                 .Select(pkg => new PackageEntry(pkg, GetPrecedence(pkg))) | ||||
|                 .OrderBy(static entry => entry.Rank) | ||||
|                 .ThenByDescending(static entry => entry.Package.Provenance.Length) | ||||
|                 .ThenByDescending(static entry => entry.Package.VersionRanges.Length) | ||||
|                 .ToList(); | ||||
|  | ||||
|             var primary = ordered.First(); | ||||
|             var primary = ordered[0]; | ||||
|             var provenance = ordered | ||||
|                 .SelectMany(static pkg => pkg.Provenance) | ||||
|                 .SelectMany(static entry => entry.Package.Provenance) | ||||
|                 .Where(static p => p is not null) | ||||
|                 .Distinct() | ||||
|                 .ToImmutableArray(); | ||||
|  | ||||
|             var statuses = ordered | ||||
|                 .SelectMany(static pkg => pkg.Statuses) | ||||
|                 .SelectMany(static entry => entry.Package.Statuses) | ||||
|                 .Distinct(AffectedPackageStatusEqualityComparer.Instance) | ||||
|                 .ToImmutableArray(); | ||||
|  | ||||
|             foreach (var candidate in ordered.Skip(1)) | ||||
|             { | ||||
|                 if (candidate.Package.VersionRanges.Length == 0) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 overrides.Add(new AffectedPackageOverride( | ||||
|                     primary.Package.Type, | ||||
|                     primary.Package.Identifier, | ||||
|                     string.IsNullOrWhiteSpace(primary.Package.Platform) ? null : primary.Package.Platform, | ||||
|                     primary.Rank, | ||||
|                     candidate.Rank, | ||||
|                     ExtractSources(primary.Package), | ||||
|                     ExtractSources(candidate.Package), | ||||
|                     primary.Package.VersionRanges.Length, | ||||
|                     candidate.Package.VersionRanges.Length)); | ||||
|             } | ||||
|  | ||||
|             var merged = new AffectedPackage( | ||||
|                 primary.Type, | ||||
|                 primary.Identifier, | ||||
|                 string.IsNullOrWhiteSpace(primary.Platform) ? null : primary.Platform, | ||||
|                 primary.VersionRanges, | ||||
|                 primary.Package.VersionRanges, | ||||
|                 statuses, | ||||
|                 provenance); | ||||
|  | ||||
|             resolved.Add(merged); | ||||
|         } | ||||
|  | ||||
|         return resolved | ||||
|         var packagesResult = resolved | ||||
|             .OrderBy(static pkg => pkg.Type, StringComparer.Ordinal) | ||||
|             .ThenBy(static pkg => pkg.Identifier, StringComparer.Ordinal) | ||||
|             .ThenBy(static pkg => pkg.Platform, StringComparer.Ordinal) | ||||
|             .ToImmutableArray(); | ||||
|  | ||||
|         return new AffectedPackagePrecedenceResult(packagesResult, overrides.ToImmutableArray()); | ||||
|     } | ||||
|  | ||||
|     private int GetPrecedence(AffectedPackage package) | ||||
| @@ -111,4 +122,42 @@ public sealed class AffectedPackagePrecedenceResolver | ||||
|  | ||||
|         return bestRank; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<string> ExtractSources(AffectedPackage package) | ||||
|     { | ||||
|         if (package.Provenance.Length == 0) | ||||
|         { | ||||
|             return Array.Empty<string>(); | ||||
|         } | ||||
|  | ||||
|         return package.Provenance | ||||
|             .Select(static p => p.Source) | ||||
|             .Where(static source => !string.IsNullOrWhiteSpace(source)) | ||||
|             .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|             .ToImmutableArray(); | ||||
|     } | ||||
|  | ||||
|     private readonly record struct PackageEntry(AffectedPackage Package, int Rank) | ||||
|     { | ||||
|         public string Type => Package.Type; | ||||
|  | ||||
|         public string Identifier => Package.Identifier; | ||||
|  | ||||
|         public string? Platform => string.IsNullOrWhiteSpace(Package.Platform) ? null : Package.Platform; | ||||
|     } | ||||
| } | ||||
|  | ||||
| public sealed record AffectedPackagePrecedenceResult( | ||||
|     IReadOnlyList<AffectedPackage> Packages, | ||||
|     IReadOnlyList<AffectedPackageOverride> Overrides); | ||||
|  | ||||
| public sealed record AffectedPackageOverride( | ||||
|     string Type, | ||||
|     string Identifier, | ||||
|     string? Platform, | ||||
|     int PrimaryRank, | ||||
|     int SuppressedRank, | ||||
|     IReadOnlyList<string> PrimarySources, | ||||
|     IReadOnlyList<string> SuppressedSources, | ||||
|     int PrimaryRangeCount, | ||||
|     int SuppressedRangeCount); | ||||
|   | ||||
							
								
								
									
										139
									
								
								src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										139
									
								
								src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,139 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Feedser.Storage.Mongo.Aliases; | ||||
|  | ||||
| namespace StellaOps.Feedser.Merge.Services; | ||||
|  | ||||
| public sealed class AliasGraphResolver | ||||
| { | ||||
|     private readonly IAliasStore _aliasStore; | ||||
|  | ||||
|     public AliasGraphResolver(IAliasStore aliasStore) | ||||
|     { | ||||
|         _aliasStore = aliasStore ?? throw new ArgumentNullException(nameof(aliasStore)); | ||||
|     } | ||||
|  | ||||
|     public async Task<AliasIdentityResult> ResolveAsync(string advisoryKey, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentException.ThrowIfNullOrEmpty(advisoryKey); | ||||
|         var aliases = await _aliasStore.GetByAdvisoryAsync(advisoryKey, cancellationToken).ConfigureAwait(false); | ||||
|         var collisions = new List<AliasCollision>(); | ||||
|  | ||||
|         foreach (var alias in aliases) | ||||
|         { | ||||
|             var candidates = await _aliasStore.GetByAliasAsync(alias.Scheme, alias.Value, cancellationToken).ConfigureAwait(false); | ||||
|             var advisoryKeys = candidates | ||||
|                 .Select(static candidate => candidate.AdvisoryKey) | ||||
|                 .Where(static key => !string.IsNullOrWhiteSpace(key)) | ||||
|                 .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|                 .ToArray(); | ||||
|  | ||||
|             if (advisoryKeys.Length <= 1) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             collisions.Add(new AliasCollision(alias.Scheme, alias.Value, advisoryKeys)); | ||||
|         } | ||||
|  | ||||
|         var unique = new Dictionary<string, AliasCollision>(StringComparer.Ordinal); | ||||
|         foreach (var collision in collisions) | ||||
|         { | ||||
|             var key = $"{collision.Scheme}\u0001{collision.Value}"; | ||||
|             if (!unique.ContainsKey(key)) | ||||
|             { | ||||
|                 unique[key] = collision; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var distinctCollisions = unique.Values.ToArray(); | ||||
|  | ||||
|         return new AliasIdentityResult(advisoryKey, aliases, distinctCollisions); | ||||
|     } | ||||
|  | ||||
|     public async Task<AliasComponent> BuildComponentAsync(string advisoryKey, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentException.ThrowIfNullOrEmpty(advisoryKey); | ||||
|  | ||||
|         var visited = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|         var queue = new Queue<string>(); | ||||
|         var collisionMap = new Dictionary<string, AliasCollision>(StringComparer.Ordinal); | ||||
|  | ||||
|         var aliasCache = new Dictionary<string, IReadOnlyList<AliasRecord>>(StringComparer.OrdinalIgnoreCase); | ||||
|         queue.Enqueue(advisoryKey); | ||||
|  | ||||
|         while (queue.Count > 0) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|             var current = queue.Dequeue(); | ||||
|             if (!visited.Add(current)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var aliases = await GetAliasesAsync(current, cancellationToken, aliasCache).ConfigureAwait(false); | ||||
|             aliasCache[current] = aliases; | ||||
|             foreach (var alias in aliases) | ||||
|             { | ||||
|                 var aliasRecords = await GetAdvisoriesForAliasAsync(alias.Scheme, alias.Value, cancellationToken).ConfigureAwait(false); | ||||
|                 var advisoryKeys = aliasRecords | ||||
|                     .Select(static record => record.AdvisoryKey) | ||||
|                     .Where(static key => !string.IsNullOrWhiteSpace(key)) | ||||
|                     .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|                     .ToArray(); | ||||
|  | ||||
|                 if (advisoryKeys.Length <= 1) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 foreach (var candidate in advisoryKeys) | ||||
|                 { | ||||
|                     if (!visited.Contains(candidate)) | ||||
|                     { | ||||
|                         queue.Enqueue(candidate); | ||||
|                     } | ||||
|                 } | ||||
|  | ||||
|                 var collision = new AliasCollision(alias.Scheme, alias.Value, advisoryKeys); | ||||
|                 var key = $"{collision.Scheme}\u0001{collision.Value}"; | ||||
|                 collisionMap.TryAdd(key, collision); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var aliasMap = new Dictionary<string, IReadOnlyList<AliasRecord>>(aliasCache, StringComparer.OrdinalIgnoreCase); | ||||
|         return new AliasComponent(advisoryKey, visited.ToArray(), collisionMap.Values.ToArray(), aliasMap); | ||||
|     } | ||||
|  | ||||
|     private async Task<IReadOnlyList<AliasRecord>> GetAliasesAsync( | ||||
|         string advisoryKey, | ||||
|         CancellationToken cancellationToken, | ||||
|         IDictionary<string, IReadOnlyList<AliasRecord>> cache) | ||||
|     { | ||||
|         if (cache.TryGetValue(advisoryKey, out var cached)) | ||||
|         { | ||||
|             return cached; | ||||
|         } | ||||
|  | ||||
|         var aliases = await _aliasStore.GetByAdvisoryAsync(advisoryKey, cancellationToken).ConfigureAwait(false); | ||||
|         cache[advisoryKey] = aliases; | ||||
|         return aliases; | ||||
|     } | ||||
|  | ||||
|     private Task<IReadOnlyList<AliasRecord>> GetAdvisoriesForAliasAsync( | ||||
|         string scheme, | ||||
|         string value, | ||||
|         CancellationToken cancellationToken) | ||||
|         => _aliasStore.GetByAliasAsync(scheme, value, cancellationToken); | ||||
| } | ||||
|  | ||||
| public sealed record AliasIdentityResult(string AdvisoryKey, IReadOnlyList<AliasRecord> Aliases, IReadOnlyList<AliasCollision> Collisions); | ||||
|  | ||||
| public sealed record AliasComponent( | ||||
|     string SeedAdvisoryKey, | ||||
|     IReadOnlyList<string> AdvisoryKeys, | ||||
|     IReadOnlyList<AliasCollision> Collisions, | ||||
|     IReadOnlyDictionary<string, IReadOnlyList<AliasRecord>> AliasMap); | ||||
| @@ -8,6 +8,7 @@ | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.0" /> | ||||
|     <PackageReference Include="Semver" Version="2.3.0" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" /> | ||||
|   | ||||
| @@ -2,12 +2,12 @@ | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|DONE – `AdvisoryIdentityResolver` builds alias-driven clusters with canonical key selection + unit coverage.| | ||||
| |Precedence policy engine|BE-Merge|Architecture|PSIRT/OVAL > NVD; CERTs enrich; KEV flag.| | ||||
| |Precedence policy engine|BE-Merge|Architecture|**DONE** – precedence defaults enforced by `AdvisoryPrecedenceMerger`/`AdvisoryPrecedenceDefaults` with distro/PSIRT overriding registry feeds and CERT/KEV enrichers.| | ||||
| |NEVRA comparer plus tests|BE-Merge (Distro WG)|Source.Distro fixtures|DONE – Added Nevra parser/comparer with tilde-aware rpm ordering and unit coverage.| | ||||
| |Debian EVR comparer plus tests|BE-Merge (Distro WG)|Debian fixtures|DONE – DebianEvr comparer mirrors dpkg ordering with tilde/epoch handling and unit coverage.| | ||||
| |SemVer range resolver plus tests|BE-Merge (OSS WG)|OSV/GHSA fixtures|DONE – SemanticVersionRangeResolver covers introduced/fixed/lastAffected semantics with SemVer ordering tests.| | ||||
| |Canonical hash and merge_event writer|BE-Merge|Models, Storage.Mongo|DONE – Hash calculator + MergeEventWriter compute canonical SHA-256 digests and persist merge events.| | ||||
| |Conflict detection and metrics|BE-Merge|Core|Counters; structured logs; traces.| | ||||
| |End-to-end determinism test|QA|Merge, key connectors|Same inputs -> same hashes.| | ||||
| |Conflict detection and metrics|BE-Merge|Core|**DONE** – merge meters emit override/conflict counters and structured audits (`AdvisoryPrecedenceMerger`).| | ||||
| |End-to-end determinism test|QA|Merge, key connectors|**DONE** – `MergePrecedenceIntegrationTests.MergePipeline_IsDeterministicAcrossRuns` guards determinism.| | ||||
| |Override audit logging|BE-Merge|Observability|DONE – override audits now emit structured logs plus bounded-tag metrics suitable for prod telemetry.| | ||||
| |Configurable precedence table|BE-Merge|Architecture|DONE – precedence options bind via feedser:merge:precedence:ranks with docs/tests covering operator workflow.| | ||||
|   | ||||
| @@ -13,7 +13,8 @@ public sealed class CanonicalExamplesTests | ||||
|     public void CanonicalExamplesMatchGoldenSnapshots() | ||||
|     { | ||||
|         Directory.CreateDirectory(FixtureRoot); | ||||
|         var updateGoldens = string.Equals(Environment.GetEnvironmentVariable(UpdateEnvVar), "1", StringComparison.OrdinalIgnoreCase); | ||||
|         var envValue = Environment.GetEnvironmentVariable(UpdateEnvVar); | ||||
|         var updateGoldens = string.Equals(envValue, "1", StringComparison.OrdinalIgnoreCase); | ||||
|         var failures = new List<string>(); | ||||
|  | ||||
|         foreach (var (name, advisory) in CanonicalExampleFactory.GetExamples()) | ||||
| @@ -36,6 +37,8 @@ public sealed class CanonicalExamplesTests | ||||
|             var expected = File.ReadAllText(fixturePath).Replace("\r\n", "\n"); | ||||
|             if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) | ||||
|             { | ||||
|                 var actualPath = Path.Combine(FixtureRoot, $"{name}.actual.json"); | ||||
|                 File.WriteAllText(actualPath, snapshot); | ||||
|                 failures.Add($"Fixture mismatch for {name}. Set {UpdateEnvVar}=1 to regenerate."); | ||||
|             } | ||||
|         } | ||||
|   | ||||
| @@ -1,4 +1,5 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using System.Text.Json; | ||||
| using StellaOps.Feedser.Models; | ||||
| @@ -62,4 +63,90 @@ public sealed class CanonicalJsonSerializerTests | ||||
|         var normalized2 = snap2.Replace("\r\n", "\n"); | ||||
|         Assert.Equal(normalized1, normalized2); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public void SerializesRangePrimitivesPayload() | ||||
|     { | ||||
|         var recordedAt = new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero); | ||||
|         var provenance = new AdvisoryProvenance("connector-x", "map", "segment-1", recordedAt); | ||||
|         var primitives = new RangePrimitives( | ||||
|             new SemVerPrimitive( | ||||
|                 Introduced: "2.0.0", | ||||
|                 IntroducedInclusive: true, | ||||
|                 Fixed: "2.3.4", | ||||
|                 FixedInclusive: false, | ||||
|                 LastAffected: "2.3.3", | ||||
|                 LastAffectedInclusive: true, | ||||
|                 ConstraintExpression: ">=2.0.0 <2.3.4"), | ||||
|             new NevraPrimitive( | ||||
|                 Introduced: new NevraComponent("pkg", 0, "2.0.0", "1", "x86_64"), | ||||
|                 Fixed: null, | ||||
|                 LastAffected: new NevraComponent("pkg", 0, "2.3.3", "3", "x86_64")), | ||||
|             new EvrPrimitive( | ||||
|                 Introduced: new EvrComponent(1, "2.0.0", "1"), | ||||
|                 Fixed: new EvrComponent(1, "2.3.4", null), | ||||
|                 LastAffected: null), | ||||
|             new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|             { | ||||
|                 ["channel"] = "stable", | ||||
|             }); | ||||
|  | ||||
|         var range = new AffectedVersionRange( | ||||
|             rangeKind: "semver", | ||||
|             introducedVersion: "2.0.0", | ||||
|             fixedVersion: "2.3.4", | ||||
|             lastAffectedVersion: "2.3.3", | ||||
|             rangeExpression: ">=2.0.0 <2.3.4", | ||||
|             provenance, | ||||
|             primitives); | ||||
|  | ||||
|         var package = new AffectedPackage( | ||||
|             type: "semver", | ||||
|             identifier: "pkg@2.x", | ||||
|             platform: "linux", | ||||
|             versionRanges: new[] { range }, | ||||
|             statuses: Array.Empty<AffectedPackageStatus>(), | ||||
|             provenance: new[] { provenance }); | ||||
|  | ||||
|         var advisory = new Advisory( | ||||
|             advisoryKey: "TEST-PRIM", | ||||
|             title: "Range primitive serialization", | ||||
|             summary: null, | ||||
|             language: null, | ||||
|             published: recordedAt, | ||||
|             modified: recordedAt, | ||||
|             severity: null, | ||||
|             exploitKnown: false, | ||||
|             aliases: Array.Empty<string>(), | ||||
|             references: Array.Empty<AdvisoryReference>(), | ||||
|             affectedPackages: new[] { package }, | ||||
|             cvssMetrics: Array.Empty<CvssMetric>(), | ||||
|             provenance: new[] { provenance }); | ||||
|  | ||||
|         var json = CanonicalJsonSerializer.Serialize(advisory); | ||||
|         using var document = JsonDocument.Parse(json); | ||||
|         var rangeElement = document.RootElement | ||||
|             .GetProperty("affectedPackages")[0] | ||||
|             .GetProperty("versionRanges")[0]; | ||||
|  | ||||
|         Assert.True(rangeElement.TryGetProperty("primitives", out var primitivesElement)); | ||||
|  | ||||
|         var semver = primitivesElement.GetProperty("semVer"); | ||||
|         Assert.Equal("2.0.0", semver.GetProperty("introduced").GetString()); | ||||
|         Assert.True(semver.GetProperty("introducedInclusive").GetBoolean()); | ||||
|         Assert.Equal("2.3.4", semver.GetProperty("fixed").GetString()); | ||||
|         Assert.False(semver.GetProperty("fixedInclusive").GetBoolean()); | ||||
|         Assert.Equal("2.3.3", semver.GetProperty("lastAffected").GetString()); | ||||
|  | ||||
|         var nevra = primitivesElement.GetProperty("nevra"); | ||||
|         Assert.Equal("pkg", nevra.GetProperty("introduced").GetProperty("name").GetString()); | ||||
|         Assert.Equal(0, nevra.GetProperty("introduced").GetProperty("epoch").GetInt32()); | ||||
|  | ||||
|         var evr = primitivesElement.GetProperty("evr"); | ||||
|         Assert.Equal(1, evr.GetProperty("introduced").GetProperty("epoch").GetInt32()); | ||||
|         Assert.Equal("2.3.4", evr.GetProperty("fixed").GetProperty("upstreamVersion").GetString()); | ||||
|  | ||||
|         var extensions = primitivesElement.GetProperty("vendorExtensions"); | ||||
|         Assert.Equal("stable", extensions.GetProperty("channel").GetString()); | ||||
|     } | ||||
| } | ||||
|   | ||||
| @@ -19,6 +19,7 @@ | ||||
|           "fixedVersion": "2.5.1", | ||||
|           "introducedVersion": null, | ||||
|           "lastAffectedVersion": null, | ||||
|           "primitives": null, | ||||
|           "provenance": { | ||||
|             "kind": "map", | ||||
|             "recordedAt": "2024-03-05T10:00:00+00:00", | ||||
| @@ -32,6 +33,7 @@ | ||||
|           "fixedVersion": "3.2.4", | ||||
|           "introducedVersion": "3.0.0", | ||||
|           "lastAffectedVersion": null, | ||||
|           "primitives": null, | ||||
|           "provenance": { | ||||
|             "kind": "map", | ||||
|             "recordedAt": "2024-03-05T10:00:00+00:00", | ||||
|   | ||||
| @@ -29,6 +29,7 @@ | ||||
|           "fixedVersion": "1.0.5", | ||||
|           "introducedVersion": "1.0", | ||||
|           "lastAffectedVersion": null, | ||||
|           "primitives": null, | ||||
|           "provenance": { | ||||
|             "kind": "map", | ||||
|             "recordedAt": "2024-08-01T12:00:00+00:00", | ||||
|   | ||||
| @@ -35,6 +35,7 @@ | ||||
|           "fixedVersion": null, | ||||
|           "introducedVersion": "0:4.18.0-553.el8", | ||||
|           "lastAffectedVersion": null, | ||||
|           "primitives": null, | ||||
|           "provenance": { | ||||
|             "kind": "map", | ||||
|             "recordedAt": "2024-05-11T09:00:00+00:00", | ||||
|   | ||||
							
								
								
									
										178
									
								
								src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										178
									
								
								src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,178 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Diagnostics.Metrics; | ||||
| using System.Linq; | ||||
| using System.Reflection; | ||||
| using Microsoft.Extensions.Logging.Abstractions; | ||||
| using StellaOps.Feedser.Models; | ||||
| using Xunit; | ||||
|  | ||||
| namespace StellaOps.Feedser.Models.Tests; | ||||
|  | ||||
| public sealed class ProvenanceDiagnosticsTests | ||||
| { | ||||
|     [Fact] | ||||
|     public void RecordMissing_AddsExpectedTagsAndDeduplicates() | ||||
|     { | ||||
|         ResetState(); | ||||
|  | ||||
|         var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary<string, object?> Tags)>(); | ||||
|         using var listener = CreateListener(measurements); | ||||
|  | ||||
|         var baseline = DateTimeOffset.UtcNow; | ||||
|         ProvenanceDiagnostics.RecordMissing("source-A", "range:pkg", baseline); | ||||
|         ProvenanceDiagnostics.RecordMissing("source-A", "range:pkg", baseline.AddMinutes(5)); | ||||
|         ProvenanceDiagnostics.RecordMissing("source-A", "reference:https://example", baseline.AddMinutes(10)); | ||||
|  | ||||
|         listener.Dispose(); | ||||
|  | ||||
|         Assert.Equal(2, measurements.Count); | ||||
|  | ||||
|         var first = measurements[0]; | ||||
|         Assert.Equal(1, first.Value); | ||||
|         Assert.Equal("feedser.provenance.missing", first.Instrument); | ||||
|         Assert.Equal("source-A", first.Tags["source"]); | ||||
|         Assert.Equal("range:pkg", first.Tags["component"]); | ||||
|         Assert.Equal("range", first.Tags["category"]); | ||||
|         Assert.Equal("high", first.Tags["severity"]); | ||||
|  | ||||
|         var second = measurements[1]; | ||||
|         Assert.Equal("feedser.provenance.missing", second.Instrument); | ||||
|         Assert.Equal("reference", second.Tags["category"]); | ||||
|         Assert.Equal("low", second.Tags["severity"]); | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public void ReportResumeWindow_ClearsTrackedEntries_WhenWindowBackfills() | ||||
|     { | ||||
|         ResetState(); | ||||
|  | ||||
|         var timestamp = DateTimeOffset.UtcNow; | ||||
|         ProvenanceDiagnostics.RecordMissing("source-B", "package:lib", timestamp); | ||||
|  | ||||
|         var (recorded, earliest, syncRoot) = GetInternalState(); | ||||
|         lock (syncRoot) | ||||
|         { | ||||
|             Assert.True(earliest.ContainsKey("source-B")); | ||||
|             Assert.Contains(recorded, entry => entry.StartsWith("source-B|", StringComparison.OrdinalIgnoreCase)); | ||||
|         } | ||||
|  | ||||
|         ProvenanceDiagnostics.ReportResumeWindow("source-B", timestamp.AddMinutes(-5), NullLogger.Instance); | ||||
|  | ||||
|         lock (syncRoot) | ||||
|         { | ||||
|             Assert.False(earliest.ContainsKey("source-B")); | ||||
|             Assert.DoesNotContain(recorded, entry => entry.StartsWith("source-B|", StringComparison.OrdinalIgnoreCase)); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public void ReportResumeWindow_RetainsEntries_WhenWindowTooRecent() | ||||
|     { | ||||
|         ResetState(); | ||||
|  | ||||
|         var timestamp = DateTimeOffset.UtcNow; | ||||
|         ProvenanceDiagnostics.RecordMissing("source-C", "range:pkg", timestamp); | ||||
|  | ||||
|         ProvenanceDiagnostics.ReportResumeWindow("source-C", timestamp.AddMinutes(1), NullLogger.Instance); | ||||
|  | ||||
|         var (recorded, earliest, syncRoot) = GetInternalState(); | ||||
|         lock (syncRoot) | ||||
|         { | ||||
|             Assert.True(earliest.ContainsKey("source-C")); | ||||
|             Assert.Contains(recorded, entry => entry.StartsWith("source-C|", StringComparison.OrdinalIgnoreCase)); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public void RecordRangePrimitive_EmitsCoverageMetric() | ||||
|     { | ||||
|         var range = new AffectedVersionRange( | ||||
|             rangeKind: "evr", | ||||
|             introducedVersion: "1:1.1.1n-0+deb11u2", | ||||
|             fixedVersion: null, | ||||
|             lastAffectedVersion: null, | ||||
|             rangeExpression: null, | ||||
|             provenance: new AdvisoryProvenance("source-D", "range", "pkg", DateTimeOffset.UtcNow), | ||||
|             primitives: new RangePrimitives( | ||||
|                 SemVer: null, | ||||
|                 Nevra: null, | ||||
|                 Evr: new EvrPrimitive( | ||||
|                     new EvrComponent(1, "1.1.1n", "0+deb11u2"), | ||||
|                     null, | ||||
|                     null), | ||||
|                 VendorExtensions: new Dictionary<string, string> { ["debian.release"] = "bullseye" })); | ||||
|  | ||||
|         var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary<string, object?> Tags)>(); | ||||
|         using var listener = CreateListener(measurements, "feedser.range.primitives"); | ||||
|  | ||||
|         ProvenanceDiagnostics.RecordRangePrimitive("source-D", range); | ||||
|  | ||||
|         listener.Dispose(); | ||||
|  | ||||
|         var record = Assert.Single(measurements); | ||||
|         Assert.Equal("feedser.range.primitives", record.Instrument); | ||||
|         Assert.Equal(1, record.Value); | ||||
|         Assert.Equal("source-D", record.Tags["source"]); | ||||
|         Assert.Equal("evr", record.Tags["rangeKind"]); | ||||
|         Assert.Equal("evr", record.Tags["primitiveKinds"]); | ||||
|         Assert.Equal("true", record.Tags["hasVendorExtensions"]); | ||||
|     } | ||||
|  | ||||
|     private static MeterListener CreateListener( | ||||
|         List<(string Instrument, long Value, IReadOnlyDictionary<string, object?> Tags)> measurements, | ||||
|         params string[] instrumentNames) | ||||
|     { | ||||
|         var allowed = instrumentNames is { Length: > 0 } ? instrumentNames : new[] { "feedser.provenance.missing" }; | ||||
|         var allowedSet = new HashSet<string>(allowed, StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         var listener = new MeterListener | ||||
|         { | ||||
|             InstrumentPublished = (instrument, l) => | ||||
|             { | ||||
|                 if (instrument.Meter.Name == "StellaOps.Feedser.Models.Provenance" && allowedSet.Contains(instrument.Name)) | ||||
|                 { | ||||
|                     l.EnableMeasurementEvents(instrument); | ||||
|                 } | ||||
|             } | ||||
|         }; | ||||
|  | ||||
|         listener.SetMeasurementEventCallback<long>((instrument, measurement, tags, state) => | ||||
|         { | ||||
|             var dict = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase); | ||||
|             foreach (var tag in tags) | ||||
|             { | ||||
|                 dict[tag.Key] = tag.Value; | ||||
|             } | ||||
|  | ||||
|             measurements.Add((instrument.Name, measurement, dict)); | ||||
|         }); | ||||
|  | ||||
|         listener.Start(); | ||||
|         return listener; | ||||
|     } | ||||
|  | ||||
|     private static void ResetState() | ||||
|     { | ||||
|         var (_, _, syncRoot) = GetInternalState(); | ||||
|         lock (syncRoot) | ||||
|         { | ||||
|             var (recorded, earliest, _) = GetInternalState(); | ||||
|             recorded.Clear(); | ||||
|             earliest.Clear(); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static (HashSet<string> Recorded, Dictionary<string, DateTimeOffset> Earliest, object SyncRoot) GetInternalState() | ||||
|     { | ||||
|         var type = typeof(ProvenanceDiagnostics); | ||||
|         var recordedField = type.GetField("RecordedComponents", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("RecordedComponents not found"); | ||||
|         var earliestField = type.GetField("EarliestMissing", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("EarliestMissing not found"); | ||||
|         var syncField = type.GetField("SyncRoot", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("SyncRoot not found"); | ||||
|  | ||||
|         var recorded = (HashSet<string>)recordedField.GetValue(null)!; | ||||
|         var earliest = (Dictionary<string, DateTimeOffset>)earliestField.GetValue(null)!; | ||||
|         var sync = syncField.GetValue(null)!; | ||||
|         return (recorded, earliest, sync); | ||||
|     } | ||||
| } | ||||
| @@ -14,7 +14,8 @@ public sealed record AffectedVersionRange | ||||
|         string? fixedVersion, | ||||
|         string? lastAffectedVersion, | ||||
|         string? rangeExpression, | ||||
|         AdvisoryProvenance provenance) | ||||
|         AdvisoryProvenance provenance, | ||||
|         RangePrimitives? primitives = null) | ||||
|     { | ||||
|         RangeKind = Validation.EnsureNotNullOrWhiteSpace(rangeKind, nameof(rangeKind)).ToLowerInvariant(); | ||||
|         IntroducedVersion = Validation.TrimToNull(introducedVersion); | ||||
| @@ -22,6 +23,7 @@ public sealed record AffectedVersionRange | ||||
|         LastAffectedVersion = Validation.TrimToNull(lastAffectedVersion); | ||||
|         RangeExpression = Validation.TrimToNull(rangeExpression); | ||||
|         Provenance = provenance ?? AdvisoryProvenance.Empty; | ||||
|         Primitives = primitives; | ||||
|     } | ||||
|  | ||||
|     /// <summary> | ||||
| @@ -51,6 +53,8 @@ public sealed record AffectedVersionRange | ||||
|  | ||||
|     public AdvisoryProvenance Provenance { get; } | ||||
|  | ||||
|     public RangePrimitives? Primitives { get; } | ||||
|  | ||||
|     public string CreateDeterministicKey() | ||||
|         => string.Join('|', RangeKind, IntroducedVersion ?? string.Empty, FixedVersion ?? string.Empty, LastAffectedVersion ?? string.Empty, RangeExpression ?? string.Empty); | ||||
| } | ||||
|   | ||||
| @@ -63,6 +63,7 @@ Deterministic ordering: packages sorted by `type`, then `identifier`, then `plat | ||||
| | `lastAffectedVersion` | string? | optional | Inclusive upper bound when no fix exists. | | ||||
| | `rangeExpression` | string? | optional | Normalized textual expression for non-simple ranges. | | ||||
| | `provenance` | AdvisoryProvenance | yes | Provenance entry for the range. | | ||||
| | `primitives` | RangePrimitives? | optional | Structured metadata (SemVer/Nevra/Evr/vendor extensions) when available. | | ||||
|  | ||||
| Comparers/equality ignore provenance differences. | ||||
|  | ||||
|   | ||||
| @@ -10,3 +10,5 @@ | ||||
| - **Merge policy**: never discard provenance when merging; instead append a new `AdvisoryProvenance` entry with the merge routine (`source=merge.determine-precedence`). | ||||
| - **Determinism**: provenance collections are sorted by source → kind → recordedAt before serialization; avoid generating random identifiers inside provenance. | ||||
| - **Redaction**: keep provenance values free of secrets; prefer tokens or normalized descriptors when referencing authenticated fetches. | ||||
| - **Range telemetry**: each `AffectedVersionRange` is observed by the `feedser.range.primitives` metric. Emit the richest `RangePrimitives` possible (SemVer/NEVRA/EVR plus vendor extensions); the telemetry tags make it easy to spot connectors missing structured range data. | ||||
| - **Vendor extensions**: when vendor feeds surface bespoke status flags, capture them in `RangePrimitives.VendorExtensions`. SUSE advisories publish `suse.status` (open/resolved/investigating) and Ubuntu notices expose `ubuntu.pocket`/`ubuntu.release` to distinguish security vs ESM pockets; Adobe APSB bulletins emit `adobe.track`, `adobe.platform`, `adobe.priority`, `adobe.availability`, plus `adobe.affected.raw`/`adobe.updated.raw` to preserve PSIRT metadata while keeping the status catalog canonical. These values are exported for dashboards and alerting. | ||||
|   | ||||
							
								
								
									
										253
									
								
								src/StellaOps.Feedser.Models/ProvenanceInspector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										253
									
								
								src/StellaOps.Feedser.Models/ProvenanceInspector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,253 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Diagnostics.Metrics; | ||||
| using System.Linq; | ||||
| using Microsoft.Extensions.Logging; | ||||
|  | ||||
| namespace StellaOps.Feedser.Models; | ||||
|  | ||||
| public static class ProvenanceInspector | ||||
| { | ||||
|     public static IReadOnlyList<MissingProvenance> FindMissingProvenance(Advisory advisory) | ||||
|     { | ||||
|         var results = new List<MissingProvenance>(); | ||||
|         var source = advisory.Provenance.FirstOrDefault()?.Source ?? "unknown"; | ||||
|  | ||||
|         if (advisory.Provenance.Length == 0) | ||||
|         { | ||||
|             results.Add(new MissingProvenance(source, "advisory", null)); | ||||
|         } | ||||
|  | ||||
|         foreach (var reference in advisory.References) | ||||
|         { | ||||
|             if (IsMissing(reference.Provenance)) | ||||
|             { | ||||
|                 results.Add(new MissingProvenance(reference.Provenance.Source ?? source, $"reference:{reference.Url}", reference.Provenance.RecordedAt)); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         foreach (var package in advisory.AffectedPackages) | ||||
|         { | ||||
|             if (package.Provenance.Length == 0) | ||||
|             { | ||||
|                 results.Add(new MissingProvenance(source, $"package:{package.Identifier}", null)); | ||||
|             } | ||||
|  | ||||
|             foreach (var range in package.VersionRanges) | ||||
|             { | ||||
|                 ProvenanceDiagnostics.RecordRangePrimitive(range.Provenance.Source ?? source, range); | ||||
|  | ||||
|                 if (IsMissing(range.Provenance)) | ||||
|                 { | ||||
|                     results.Add(new MissingProvenance(range.Provenance.Source ?? source, $"range:{package.Identifier}", range.Provenance.RecordedAt)); | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             foreach (var status in package.Statuses) | ||||
|             { | ||||
|                 if (IsMissing(status.Provenance)) | ||||
|                 { | ||||
|                     results.Add(new MissingProvenance(status.Provenance.Source ?? source, $"status:{package.Identifier}:{status.Status}", status.Provenance.RecordedAt)); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         foreach (var metric in advisory.CvssMetrics) | ||||
|         { | ||||
|             if (IsMissing(metric.Provenance)) | ||||
|             { | ||||
|                 results.Add(new MissingProvenance(metric.Provenance.Source ?? source, $"cvss:{metric.Version}", metric.Provenance.RecordedAt)); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return results; | ||||
|     } | ||||
|  | ||||
|     private static bool IsMissing(AdvisoryProvenance provenance) | ||||
|     { | ||||
|         return provenance == AdvisoryProvenance.Empty | ||||
|             || string.IsNullOrWhiteSpace(provenance.Source) | ||||
|             || string.IsNullOrWhiteSpace(provenance.Kind); | ||||
|     } | ||||
| } | ||||
|  | ||||
| public sealed record MissingProvenance(string Source, string Component, DateTimeOffset? RecordedAt); | ||||
|  | ||||
| public static class ProvenanceDiagnostics | ||||
| { | ||||
|     private static readonly Meter Meter = new("StellaOps.Feedser.Models.Provenance"); | ||||
|     private static readonly Counter<long> MissingCounter = Meter.CreateCounter<long>( | ||||
|         "feedser.provenance.missing", | ||||
|         unit: "count", | ||||
|         description: "Number of canonical objects missing provenance metadata."); | ||||
|     private static readonly Counter<long> RangePrimitiveCounter = Meter.CreateCounter<long>( | ||||
|         "feedser.range.primitives", | ||||
|         unit: "count", | ||||
|         description: "Range coverage by kind, primitive availability, and vendor extensions."); | ||||
|  | ||||
|     private static readonly object SyncRoot = new(); | ||||
|     private static readonly Dictionary<string, DateTimeOffset> EarliestMissing = new(StringComparer.OrdinalIgnoreCase); | ||||
|     private static readonly HashSet<string> RecordedComponents = new(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|     public static void RecordMissing(string source, string component, DateTimeOffset? recordedAt) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(source)) | ||||
|         { | ||||
|             source = "unknown"; | ||||
|         } | ||||
|  | ||||
|         component = string.IsNullOrWhiteSpace(component) ? "unknown" : component.Trim(); | ||||
|  | ||||
|         bool shouldRecord; | ||||
|         lock (SyncRoot) | ||||
|         { | ||||
|             var key = $"{source}|{component}"; | ||||
|             shouldRecord = RecordedComponents.Add(key); | ||||
|  | ||||
|             if (recordedAt.HasValue) | ||||
|             { | ||||
|                 if (!EarliestMissing.TryGetValue(source, out var existing) || recordedAt.Value < existing) | ||||
|                 { | ||||
|                     EarliestMissing[source] = recordedAt.Value; | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (!shouldRecord) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var category = DetermineCategory(component); | ||||
|         var severity = DetermineSeverity(category); | ||||
|  | ||||
|         var tags = new[] | ||||
|         { | ||||
|             new KeyValuePair<string, object?>("source", source), | ||||
|             new KeyValuePair<string, object?>("component", component), | ||||
|             new KeyValuePair<string, object?>("category", category), | ||||
|             new KeyValuePair<string, object?>("severity", severity), | ||||
|         }; | ||||
|         MissingCounter.Add(1, tags); | ||||
|     } | ||||
|  | ||||
|     public static void ReportResumeWindow(string source, DateTimeOffset windowStart, ILogger logger) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(source) || logger is null) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         DateTimeOffset earliest; | ||||
|         var hasEntry = false; | ||||
|         lock (SyncRoot) | ||||
|         { | ||||
|             if (EarliestMissing.TryGetValue(source, out earliest)) | ||||
|             { | ||||
|                 hasEntry = true; | ||||
|                 if (windowStart <= earliest) | ||||
|                 { | ||||
|                     EarliestMissing.Remove(source); | ||||
|                     var prefix = source + "|"; | ||||
|                     RecordedComponents.RemoveWhere(entry => entry.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (!hasEntry) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         if (windowStart <= earliest) | ||||
|         { | ||||
|             logger.LogInformation( | ||||
|                 "Resume window starting {WindowStart:o} for {Source} may backfill missing provenance recorded at {Earliest:o}.", | ||||
|                 windowStart, | ||||
|                 source, | ||||
|                 earliest); | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             logger.LogInformation( | ||||
|                 "Earliest missing provenance for {Source} remains at {Earliest:o}; current resume window begins at {WindowStart:o}. Consider widening overlap to backfill.", | ||||
|                 source, | ||||
|                 earliest, | ||||
|                 windowStart); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public static void RecordRangePrimitive(string source, AffectedVersionRange range) | ||||
|     { | ||||
|         if (range is null) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         source = string.IsNullOrWhiteSpace(source) ? "unknown" : source.Trim(); | ||||
|  | ||||
|         var primitives = range.Primitives; | ||||
|         var primitiveKinds = DeterminePrimitiveKinds(primitives); | ||||
|         var vendorExtensions = primitives?.VendorExtensions?.Count ?? 0; | ||||
|  | ||||
|         var tags = new[] | ||||
|         { | ||||
|             new KeyValuePair<string, object?>("source", source), | ||||
|             new KeyValuePair<string, object?>("rangeKind", string.IsNullOrWhiteSpace(range.RangeKind) ? "unknown" : range.RangeKind), | ||||
|             new KeyValuePair<string, object?>("primitiveKinds", primitiveKinds), | ||||
|             new KeyValuePair<string, object?>("hasVendorExtensions", vendorExtensions > 0 ? "true" : "false"), | ||||
|         }; | ||||
|  | ||||
|         RangePrimitiveCounter.Add(1, tags); | ||||
|     } | ||||
|  | ||||
|     private static string DetermineCategory(string component) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(component)) | ||||
|         { | ||||
|             return "unknown"; | ||||
|         } | ||||
|  | ||||
|         var index = component.IndexOf(':'); | ||||
|         var category = index > 0 ? component[..index] : component; | ||||
|         return category.Trim().ToLowerInvariant(); | ||||
|     } | ||||
|  | ||||
|     private static string DetermineSeverity(string category) | ||||
|         => category switch | ||||
|         { | ||||
|             "advisory" => "critical", | ||||
|             "package" => "high", | ||||
|             "range" => "high", | ||||
|             "status" => "medium", | ||||
|             "cvss" => "medium", | ||||
|             "reference" => "low", | ||||
|             _ => "info", | ||||
|         }; | ||||
|  | ||||
|     private static string DeterminePrimitiveKinds(RangePrimitives? primitives) | ||||
|     { | ||||
|         if (primitives is null) | ||||
|         { | ||||
|             return "none"; | ||||
|         } | ||||
|  | ||||
|         var kinds = new List<string>(3); | ||||
|         if (primitives.SemVer is not null) | ||||
|         { | ||||
|             kinds.Add("semver"); | ||||
|         } | ||||
|  | ||||
|         if (primitives.Nevra is not null) | ||||
|         { | ||||
|             kinds.Add("nevra"); | ||||
|         } | ||||
|  | ||||
|         if (primitives.Evr is not null) | ||||
|         { | ||||
|             kinds.Add("evr"); | ||||
|         } | ||||
|  | ||||
|         return kinds.Count == 0 ? "vendor" : string.Join('+', kinds); | ||||
|     } | ||||
| } | ||||
							
								
								
									
										58
									
								
								src/StellaOps.Feedser.Models/RangePrimitives.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										58
									
								
								src/StellaOps.Feedser.Models/RangePrimitives.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,58 @@ | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Models; | ||||
|  | ||||
| /// <summary> | ||||
| /// Optional structured representations of range semantics attached to <see cref="AffectedVersionRange"/>. | ||||
| /// </summary> | ||||
| public sealed record RangePrimitives( | ||||
|     SemVerPrimitive? SemVer, | ||||
|     NevraPrimitive? Nevra, | ||||
|     EvrPrimitive? Evr, | ||||
|     IReadOnlyDictionary<string, string>? VendorExtensions); | ||||
|  | ||||
| /// <summary> | ||||
| /// Structured SemVer metadata for a version range. | ||||
| /// </summary> | ||||
| public sealed record SemVerPrimitive( | ||||
|     string? Introduced, | ||||
|     bool IntroducedInclusive, | ||||
|     string? Fixed, | ||||
|     bool FixedInclusive, | ||||
|     string? LastAffected, | ||||
|     bool LastAffectedInclusive, | ||||
|     string? ConstraintExpression); | ||||
|  | ||||
| /// <summary> | ||||
| /// Structured NEVRA metadata for a version range. | ||||
| /// </summary> | ||||
| public sealed record NevraPrimitive( | ||||
|     NevraComponent? Introduced, | ||||
|     NevraComponent? Fixed, | ||||
|     NevraComponent? LastAffected); | ||||
|  | ||||
| /// <summary> | ||||
| /// Structured Debian EVR metadata for a version range. | ||||
| /// </summary> | ||||
| public sealed record EvrPrimitive( | ||||
|     EvrComponent? Introduced, | ||||
|     EvrComponent? Fixed, | ||||
|     EvrComponent? LastAffected); | ||||
|  | ||||
| /// <summary> | ||||
| /// Normalized NEVRA component. | ||||
| /// </summary> | ||||
| public sealed record NevraComponent( | ||||
|     string Name, | ||||
|     int Epoch, | ||||
|     string Version, | ||||
|     string Release, | ||||
|     string? Architecture); | ||||
|  | ||||
| /// <summary> | ||||
| /// Normalized EVR component (epoch:upstream revision). | ||||
| /// </summary> | ||||
| public sealed record EvrComponent( | ||||
|     int Epoch, | ||||
|     string UpstreamVersion, | ||||
|     string? Revision); | ||||
| @@ -6,4 +6,7 @@ | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <TreatWarningsAsErrors>true</TreatWarningsAsErrors> | ||||
|   </PropertyGroup> | ||||
|   <ItemGroup> | ||||
|     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" /> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
|   | ||||
| @@ -9,8 +9,8 @@ | ||||
| |Docs: field provenance guidelines|BE-Merge|Models|DONE – see `PROVENANCE_GUIDELINES.md`.| | ||||
| |Canonical record definitions kept in sync|BE-Merge|Models|DONE – documented in `CANONICAL_RECORDS.md`; update alongside model changes.| | ||||
| |Alias scheme registry and validation helpers|BE-Merge|Models|DONE – see `AliasSchemes` & `AliasSchemeRegistry` plus validation integration/tests.| | ||||
| |Range primitives for SemVer/EVR/NEVRA metadata|BE-Merge|Models|TODO – keep structured values without parsing logic; ensure merge/export parity.| | ||||
| |Provenance envelope field masks|BE-Merge|Models|TODO – guarantee traceability for each mapped field.| | ||||
| |Range primitives for SemVer/EVR/NEVRA metadata|BE-Merge|Models|DOING – envelope + AdvisoryStore deserialisation landed; VMware/Oracle/Chromium/NVD emit primitives. Remaining connectors (Debian, SUSE, Ubuntu, Apple, Adobe, etc.) still need structured coverage + EVR population.| | ||||
| |Provenance envelope field masks|BE-Merge|Models|DOING – add richer metric tags (component category/severity), dedupe missing counts, propagate resume logging across connectors.| | ||||
| |Backward-compatibility playbook|BE-Merge, QA|Models|DONE – see `BACKWARD_COMPATIBILITY.md` for evolution policy/test checklist.| | ||||
| |Golden canonical examples|QA|Models|DONE – added `/p:UpdateGoldens=true` test hook wiring `UPDATE_GOLDENS=1` so canonical fixtures regenerate via `dotnet test`; docs/tests unchanged.| | ||||
| |Serialization determinism regression tests|QA|Models|DONE – locale-stability tests hash canonical serializer output across multiple cultures and runs.| | ||||
|   | ||||
| @@ -289,8 +289,15 @@ public sealed class CertFrConnectorTests : IAsyncLifetime | ||||
|  | ||||
|     private static string ReadFixture(string filename) | ||||
|     { | ||||
|         var path = Path.Combine(AppContext.BaseDirectory, "Source", "CertFr", "Fixtures", filename); | ||||
|         return File.ReadAllText(path); | ||||
|         var baseDirectory = AppContext.BaseDirectory; | ||||
|         var primary = Path.Combine(baseDirectory, "Source", "CertFr", "Fixtures", filename); | ||||
|         if (File.Exists(primary)) | ||||
|         { | ||||
|             return File.ReadAllText(primary); | ||||
|         } | ||||
|  | ||||
|         var fallback = Path.Combine(baseDirectory, "CertFr", "Fixtures", filename); | ||||
|         return File.ReadAllText(fallback); | ||||
|     } | ||||
|  | ||||
|     private static string Normalize(string value) | ||||
|   | ||||
| @@ -19,10 +19,9 @@ ANSSI CERT-FR advisories connector (avis/alertes) providing national enrichment: | ||||
| In: advisory metadata extraction, references, severity text, watermarking. | ||||
| Out: OVAL or package-level authority. | ||||
| ## Observability & security expectations | ||||
| - Metrics: certfr.fetch.items, certfr.parse.fail, certfr.map.count. | ||||
| - Metrics: SourceDiagnostics emits shared `feedser.source.http.*` counters/histograms tagged `feedser.source=certfr`, covering fetch counts, parse failures, and map activity. | ||||
| - Logs: feed URL(s), item ids/urls, extraction durations; no PII; allowlist hostnames. | ||||
| ## Tests | ||||
| - Author and review coverage in `../StellaOps.Feedser.Source.CertFr.Tests`. | ||||
| - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. | ||||
| - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. | ||||
|  | ||||
|   | ||||
| @@ -84,7 +84,8 @@ public sealed class CertInConnectorTests : IAsyncLifetime | ||||
|         var normalizedActual = NormalizeLineEndings(canonical); | ||||
|         if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) | ||||
|         { | ||||
|             var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "CertIn", "Fixtures", "expected-advisory.actual.json"); | ||||
|             var actualPath = ResolveFixturePath("expected-advisory.actual.json"); | ||||
|             Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); | ||||
|             File.WriteAllText(actualPath, canonical); | ||||
|         } | ||||
|  | ||||
| @@ -316,9 +317,18 @@ public sealed class CertInConnectorTests : IAsyncLifetime | ||||
|         => _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); | ||||
|  | ||||
|     private static string ReadFixture(string filename) | ||||
|         => File.ReadAllText(ResolveFixturePath(filename)); | ||||
|  | ||||
|     private static string ResolveFixturePath(string filename) | ||||
|     { | ||||
|         var path = Path.Combine(AppContext.BaseDirectory, "Source", "CertIn", "Fixtures", filename); | ||||
|         return File.ReadAllText(path); | ||||
|         var baseDirectory = AppContext.BaseDirectory; | ||||
|         var primary = Path.Combine(baseDirectory, "Source", "CertIn", "Fixtures", filename); | ||||
|         if (File.Exists(primary) || filename.EndsWith(".actual.json", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return primary; | ||||
|         } | ||||
|  | ||||
|         return Path.Combine(baseDirectory, "CertIn", "Fixtures", filename); | ||||
|     } | ||||
|  | ||||
|     private static string NormalizeLineEndings(string value) | ||||
|   | ||||
| @@ -94,4 +94,4 @@ | ||||
|   "severity": "high", | ||||
|   "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).", | ||||
|   "title": "Multiple vulnerabilities in Example Gateway" | ||||
| } | ||||
| } | ||||
| @@ -20,10 +20,9 @@ CERT-In national CERT connector; enrichment advisories for India; maps CVE lists | ||||
| In: enrichment, aliasing where stable, references, mitigation text. | ||||
| Out: package range authority; scraping behind auth walls. | ||||
| ## Observability & security expectations | ||||
| - Metrics: certin.fetch.items, certin.parse.fail, certin.map.enriched_count. | ||||
| - Metrics: shared `feedser.source.http.*` counters/histograms from SourceDiagnostics tagged `feedser.source=certin` capture fetch volume, parse failures, and map enrich counts. | ||||
| - Logs: advisory codes, CVE counts per advisory, timing; allowlist host; redact personal data if present. | ||||
| ## Tests | ||||
| - Author and review coverage in `../StellaOps.Feedser.Source.CertIn.Tests`. | ||||
| - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. | ||||
| - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. | ||||
|  | ||||
|   | ||||
| @@ -22,11 +22,10 @@ Shared connector toolkit. Provides HTTP clients, retry/backoff, conditional GET | ||||
| In: HTTP plumbing, validators, cursor/backoff utilities, hashing. | ||||
| Out: connector-specific schemas/mapping rules, merge precedence. | ||||
| ## Observability & security expectations | ||||
| - Metrics: http.req.count, http.retry.count, rate_limit.remaining, validator.fail.count. | ||||
| - Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged with `feedser.source=<connector>` plus retries/failures; connector dashboards slice on that tag instead of bespoke metric names. | ||||
| - Logs include uri, status, retries, etag; redact tokens and auth headers. | ||||
| - Distributed tracing hooks and per-connector counters should be wired centrally for consistent observability. | ||||
| ## Tests | ||||
| - Author and review coverage in `../StellaOps.Feedser.Source.Common.Tests`. | ||||
| - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. | ||||
| - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. | ||||
|  | ||||
|   | ||||
| @@ -0,0 +1,250 @@ | ||||
| using System; | ||||
| using System.IO; | ||||
| using System.Linq; | ||||
| using System.Net; | ||||
| using System.Net.Http; | ||||
| using System.Net.Http.Headers; | ||||
| using System.Text; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Http; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Logging.Abstractions; | ||||
| using Microsoft.Extensions.Options; | ||||
| using Microsoft.Extensions.Time.Testing; | ||||
| using MongoDB.Driver; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Source.Common; | ||||
| using StellaOps.Feedser.Source.Common.Testing; | ||||
| using StellaOps.Feedser.Source.Distro.Debian.Configuration; | ||||
| using StellaOps.Feedser.Storage.Mongo; | ||||
| using StellaOps.Feedser.Storage.Mongo.Advisories; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
| using StellaOps.Feedser.Storage.Mongo.Dtos; | ||||
| using StellaOps.Feedser.Testing; | ||||
| using Xunit; | ||||
| using Xunit.Abstractions; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Tests; | ||||
|  | ||||
| [Collection("mongo-fixture")] | ||||
| public sealed class DebianConnectorTests : IAsyncLifetime | ||||
| { | ||||
|     private static readonly Uri ListUri = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list"); | ||||
|     private static readonly Uri DetailResolved = new("https://security-tracker.debian.org/tracker/DSA-2024-123"); | ||||
|     private static readonly Uri DetailOpen = new("https://security-tracker.debian.org/tracker/DSA-2024-124"); | ||||
|  | ||||
|     private readonly MongoIntegrationFixture _fixture; | ||||
|     private readonly FakeTimeProvider _timeProvider; | ||||
|     private readonly CannedHttpMessageHandler _handler; | ||||
|     private readonly ITestOutputHelper _output; | ||||
|  | ||||
|     public DebianConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) | ||||
|     { | ||||
|         _fixture = fixture; | ||||
|         _handler = new CannedHttpMessageHandler(); | ||||
|         _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 12, 0, 0, 0, TimeSpan.Zero)); | ||||
|         _output = output; | ||||
|     } | ||||
|  | ||||
|     [Fact] | ||||
|     public async Task FetchParseMap_PopulatesRangePrimitivesAndResumesWithNotModified() | ||||
|     { | ||||
|         await using var provider = await BuildServiceProviderAsync(); | ||||
|  | ||||
|         SeedInitialResponses(); | ||||
|  | ||||
|         var connector = provider.GetRequiredService<DebianConnector>(); | ||||
|         await connector.FetchAsync(provider, CancellationToken.None); | ||||
|         _timeProvider.Advance(TimeSpan.FromMinutes(1)); | ||||
|         await connector.ParseAsync(provider, CancellationToken.None); | ||||
|         await connector.MapAsync(provider, CancellationToken.None); | ||||
|  | ||||
|         var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); | ||||
|         var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); | ||||
|         Assert.Equal(2, advisories.Count); | ||||
|  | ||||
|         var resolved = advisories.Single(a => a.AdvisoryKey == "DSA-2024-123"); | ||||
|         Assert.Contains("CVE-2024-1000", resolved.Aliases); | ||||
|         Assert.Contains("CVE-2024-1001", resolved.Aliases); | ||||
|         var resolvedBookworm = Assert.Single(resolved.AffectedPackages, p => p.Platform == "bookworm"); | ||||
|         var resolvedRange = Assert.Single(resolvedBookworm.VersionRanges); | ||||
|         Assert.Equal("evr", resolvedRange.RangeKind); | ||||
|         Assert.Equal("1:1.1.1n-0+deb11u2", resolvedRange.IntroducedVersion); | ||||
|         Assert.Equal("1:1.1.1n-0+deb11u5", resolvedRange.FixedVersion); | ||||
|         Assert.NotNull(resolvedRange.Primitives); | ||||
|         Assert.NotNull(resolvedRange.Primitives!.Evr); | ||||
|         Assert.Equal(1, resolvedRange.Primitives.Evr!.Introduced!.Epoch); | ||||
|         Assert.Equal("1.1.1n", resolvedRange.Primitives.Evr.Introduced.UpstreamVersion); | ||||
|  | ||||
|         var open = advisories.Single(a => a.AdvisoryKey == "DSA-2024-124"); | ||||
|         Assert.Contains("CVE-2024-2000", open.Aliases); | ||||
|         var openBookworm = Assert.Single(open.AffectedPackages, p => p.Platform == "bookworm"); | ||||
|         var openRange = Assert.Single(openBookworm.VersionRanges); | ||||
|         Assert.Equal("evr", openRange.RangeKind); | ||||
|         Assert.Equal("1:1.3.1-1", openRange.IntroducedVersion); | ||||
|         Assert.Null(openRange.FixedVersion); | ||||
|         Assert.NotNull(openRange.Primitives); | ||||
|         Assert.NotNull(openRange.Primitives!.Evr); | ||||
|  | ||||
|         // Ensure data persisted through Mongo round-trip. | ||||
|         var found = await advisoryStore.FindAsync("DSA-2024-123", CancellationToken.None); | ||||
|         Assert.NotNull(found); | ||||
|         var persistedRange = Assert.Single(found!.AffectedPackages, pkg => pkg.Platform == "bookworm").VersionRanges.Single(); | ||||
|         Assert.NotNull(persistedRange.Primitives); | ||||
|         Assert.NotNull(persistedRange.Primitives!.Evr); | ||||
|  | ||||
|         // Second run should issue conditional requests and no additional parsing/mapping. | ||||
|         SeedNotModifiedResponses(); | ||||
|         await connector.FetchAsync(provider, CancellationToken.None); | ||||
|         _timeProvider.Advance(TimeSpan.FromMinutes(1)); | ||||
|         await connector.ParseAsync(provider, CancellationToken.None); | ||||
|         await connector.MapAsync(provider, CancellationToken.None); | ||||
|  | ||||
|         var documents = provider.GetRequiredService<IDocumentStore>(); | ||||
|         var listDoc = await documents.FindBySourceAndUriAsync(DebianConnectorPlugin.SourceName, DetailResolved.ToString(), CancellationToken.None); | ||||
|         Assert.NotNull(listDoc); | ||||
|  | ||||
|         var refreshed = await advisoryStore.GetRecentAsync(10, CancellationToken.None); | ||||
|         Assert.Equal(2, refreshed.Count); | ||||
|         _handler.AssertNoPendingResponses(); | ||||
|     } | ||||
|  | ||||
|     private async Task<ServiceProvider> BuildServiceProviderAsync() | ||||
|     { | ||||
|         await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName, CancellationToken.None); | ||||
|         _handler.Clear(); | ||||
|  | ||||
|         var services = new ServiceCollection(); | ||||
|         services.AddLogging(builder => builder.AddProvider(new TestOutputLoggerProvider(_output))); | ||||
|         services.AddSingleton<TimeProvider>(_timeProvider); | ||||
|         services.AddSingleton(_handler); | ||||
|  | ||||
|         services.AddMongoStorage(options => | ||||
|         { | ||||
|             options.ConnectionString = _fixture.Runner.ConnectionString; | ||||
|             options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; | ||||
|             options.CommandTimeout = TimeSpan.FromSeconds(5); | ||||
|         }); | ||||
|  | ||||
|         services.AddSourceCommon(); | ||||
|         services.AddDebianConnector(options => | ||||
|         { | ||||
|             options.ListEndpoint = ListUri; | ||||
|             options.DetailBaseUri = new Uri("https://security-tracker.debian.org/tracker/"); | ||||
|             options.MaxAdvisoriesPerFetch = 10; | ||||
|             options.RequestDelay = TimeSpan.Zero; | ||||
|         }); | ||||
|  | ||||
|         services.Configure<HttpClientFactoryOptions>(DebianOptions.HttpClientName, builderOptions => | ||||
|         { | ||||
|             builderOptions.HttpMessageHandlerBuilderActions.Add(builder => | ||||
|             { | ||||
|                 builder.PrimaryHandler = _handler; | ||||
|             }); | ||||
|         }); | ||||
|  | ||||
|         var provider = services.BuildServiceProvider(); | ||||
|         var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); | ||||
|         await bootstrapper.InitializeAsync(CancellationToken.None); | ||||
|         return provider; | ||||
|     } | ||||
|  | ||||
|     private void SeedInitialResponses() | ||||
|     { | ||||
|         AddListResponse("debian-list.txt", "\"list-v1\""); | ||||
|         AddDetailResponse(DetailResolved, "debian-detail-dsa-2024-123.html", "\"detail-123\""); | ||||
|         AddDetailResponse(DetailOpen, "debian-detail-dsa-2024-124.html", "\"detail-124\""); | ||||
|     } | ||||
|  | ||||
|     private void SeedNotModifiedResponses() | ||||
|     { | ||||
|         AddNotModifiedResponse(ListUri, "\"list-v1\""); | ||||
|         AddNotModifiedResponse(DetailResolved, "\"detail-123\""); | ||||
|         AddNotModifiedResponse(DetailOpen, "\"detail-124\""); | ||||
|     } | ||||
|  | ||||
|     private void AddListResponse(string fixture, string etag) | ||||
|     { | ||||
|         _handler.AddResponse(ListUri, () => | ||||
|         { | ||||
|             var response = new HttpResponseMessage(HttpStatusCode.OK) | ||||
|             { | ||||
|                 Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/plain"), | ||||
|             }; | ||||
|             response.Headers.ETag = new EntityTagHeaderValue(etag); | ||||
|             return response; | ||||
|         }); | ||||
|     } | ||||
|  | ||||
|     private void AddDetailResponse(Uri uri, string fixture, string etag) | ||||
|     { | ||||
|         _handler.AddResponse(uri, () => | ||||
|         { | ||||
|             var response = new HttpResponseMessage(HttpStatusCode.OK) | ||||
|             { | ||||
|                 Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), | ||||
|             }; | ||||
|             response.Headers.ETag = new EntityTagHeaderValue(etag); | ||||
|             return response; | ||||
|         }); | ||||
|     } | ||||
|  | ||||
|     private void AddNotModifiedResponse(Uri uri, string etag) | ||||
|     { | ||||
|         _handler.AddResponse(uri, request => | ||||
|         { | ||||
|             var response = new HttpResponseMessage(HttpStatusCode.NotModified); | ||||
|             response.Headers.ETag = new EntityTagHeaderValue(etag); | ||||
|             return response; | ||||
|         }); | ||||
|     } | ||||
|  | ||||
|     private static string ReadFixture(string filename) | ||||
|     { | ||||
|         var primary = Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "Debian", "Fixtures", filename); | ||||
|         if (File.Exists(primary)) | ||||
|         { | ||||
|             return File.ReadAllText(primary); | ||||
|         } | ||||
|  | ||||
|         throw new FileNotFoundException($"Fixture '{filename}' not found", filename); | ||||
|     } | ||||
|  | ||||
|     public Task InitializeAsync() => Task.CompletedTask; | ||||
|  | ||||
|     public Task DisposeAsync() => Task.CompletedTask; | ||||
|  | ||||
|     private sealed class TestOutputLoggerProvider : ILoggerProvider | ||||
|     { | ||||
|         private readonly ITestOutputHelper _output; | ||||
|  | ||||
|         public TestOutputLoggerProvider(ITestOutputHelper output) => _output = output; | ||||
|  | ||||
|         public ILogger CreateLogger(string categoryName) => new TestOutputLogger(_output); | ||||
|  | ||||
|         public void Dispose() | ||||
|         { | ||||
|         } | ||||
|  | ||||
|         private sealed class TestOutputLogger : ILogger | ||||
|         { | ||||
|             private readonly ITestOutputHelper _output; | ||||
|  | ||||
|             public TestOutputLogger(ITestOutputHelper output) => _output = output; | ||||
|  | ||||
|             public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullLogger.Instance.BeginScope(state); | ||||
|  | ||||
|             public bool IsEnabled(LogLevel logLevel) => false; | ||||
|  | ||||
|             public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) | ||||
|             { | ||||
|                 if (IsEnabled(logLevel)) | ||||
|                 { | ||||
|                     _output.WriteLine(formatter(state, exception)); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,88 @@ | ||||
| using System; | ||||
| using Xunit; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Source.Distro.Debian; | ||||
| using StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Tests; | ||||
|  | ||||
| public sealed class DebianMapperTests | ||||
| { | ||||
|     [Fact] | ||||
|     public void Map_BuildsRangePrimitives_ForResolvedPackage() | ||||
|     { | ||||
|         var dto = new DebianAdvisoryDto( | ||||
|             AdvisoryId: "DSA-2024-123", | ||||
|             SourcePackage: "openssl", | ||||
|             Title: "Openssl security update", | ||||
|             Description: "Fixes multiple issues.", | ||||
|             CveIds: new[] { "CVE-2024-1000", "CVE-2024-1001" }, | ||||
|             Packages: new[] | ||||
|             { | ||||
|                 new DebianPackageStateDto( | ||||
|                     Package: "openssl", | ||||
|                     Release: "bullseye", | ||||
|                     Status: "resolved", | ||||
|                     IntroducedVersion: "1:1.1.1n-0+deb11u2", | ||||
|                     FixedVersion: "1:1.1.1n-0+deb11u5", | ||||
|                     LastAffectedVersion: null, | ||||
|                     Published: new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero)), | ||||
|                 new DebianPackageStateDto( | ||||
|                     Package: "openssl", | ||||
|                     Release: "bookworm", | ||||
|                     Status: "open", | ||||
|                     IntroducedVersion: null, | ||||
|                     FixedVersion: null, | ||||
|                     LastAffectedVersion: null, | ||||
|                     Published: null) | ||||
|             }, | ||||
|             References: new[] | ||||
|             { | ||||
|                 new DebianReferenceDto( | ||||
|                     Url: "https://security-tracker.debian.org/tracker/DSA-2024-123", | ||||
|                     Kind: "advisory", | ||||
|                     Title: "Debian Security Advisory 2024-123"), | ||||
|             }); | ||||
|  | ||||
|         var document = new DocumentRecord( | ||||
|             Id: Guid.NewGuid(), | ||||
|             SourceName: DebianConnectorPlugin.SourceName, | ||||
|             Uri: "https://security-tracker.debian.org/tracker/DSA-2024-123", | ||||
|             FetchedAt: new DateTimeOffset(2024, 9, 1, 1, 0, 0, TimeSpan.Zero), | ||||
|             Sha256: "sha", | ||||
|             Status: "Fetched", | ||||
|             ContentType: "application/json", | ||||
|             Headers: null, | ||||
|             Metadata: null, | ||||
|             Etag: null, | ||||
|             LastModified: null, | ||||
|             GridFsId: null); | ||||
|  | ||||
|         Advisory advisory = DebianMapper.Map(dto, document, new DateTimeOffset(2024, 9, 1, 2, 0, 0, TimeSpan.Zero)); | ||||
|  | ||||
|         Assert.Equal("DSA-2024-123", advisory.AdvisoryKey); | ||||
|         Assert.Contains("CVE-2024-1000", advisory.Aliases); | ||||
|         Assert.Contains("CVE-2024-1001", advisory.Aliases); | ||||
|  | ||||
|         var resolvedPackage = Assert.Single(advisory.AffectedPackages, p => p.Platform == "bullseye"); | ||||
|         var range = Assert.Single(resolvedPackage.VersionRanges); | ||||
|         Assert.Equal("evr", range.RangeKind); | ||||
|         Assert.Equal("1:1.1.1n-0+deb11u2", range.IntroducedVersion); | ||||
|         Assert.Equal("1:1.1.1n-0+deb11u5", range.FixedVersion); | ||||
|         Assert.NotNull(range.Primitives); | ||||
|         var evr = range.Primitives!.Evr; | ||||
|         Assert.NotNull(evr); | ||||
|         Assert.NotNull(evr!.Introduced); | ||||
|         Assert.Equal(1, evr.Introduced!.Epoch); | ||||
|         Assert.Equal("1.1.1n", evr.Introduced.UpstreamVersion); | ||||
|         Assert.Equal("0+deb11u2", evr.Introduced.Revision); | ||||
|         Assert.NotNull(evr.Fixed); | ||||
|         Assert.Equal(1, evr.Fixed!.Epoch); | ||||
|         Assert.Equal("1.1.1n", evr.Fixed.UpstreamVersion); | ||||
|         Assert.Equal("0+deb11u5", evr.Fixed.Revision); | ||||
|  | ||||
|         var openPackage = Assert.Single(advisory.AffectedPackages, p => p.Platform == "bookworm"); | ||||
|         Assert.Empty(openPackage.VersionRanges); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,23 @@ | ||||
| <!DOCTYPE html> | ||||
| <html> | ||||
|   <head> | ||||
|     <title>DSA-2024-123</title> | ||||
|   </head> | ||||
|   <body> | ||||
|     <header><h1>DSA-2024-123</h1></header> | ||||
|     <table> | ||||
|       <tr><td><b>Name</b></td><td>DSA-2024-123</td></tr> | ||||
|       <tr><td><b>Description</b></td><td>openssl - security update</td></tr> | ||||
|       <tr><td><b>Source</b></td><td><a href="https://www.debian.org/security/dsa-2024-123">Debian</a></td></tr> | ||||
|       <tr><td><b>References</b></td><td><a href="/tracker/CVE-2024-1000">CVE-2024-1000</a>, <a href="/tracker/CVE-2024-1001">CVE-2024-1001</a></td></tr> | ||||
|     </table> | ||||
|     <h2>Vulnerable and fixed packages</h2> | ||||
|     <table> | ||||
|       <tr><th>Source Package</th><th>Release</th><th>Version</th><th>Status</th></tr> | ||||
|       <tr><td><a href="/tracker/source-package/openssl">openssl</a></td><td>bookworm</td><td><span class="red">1:1.1.1n-0+deb11u2</span></td><td><span class="red">vulnerable</span></td></tr> | ||||
|       <tr><td></td><td>bookworm (security)</td><td>1:1.1.1n-0+deb11u5</td><td>fixed</td></tr> | ||||
|       <tr><td></td><td>trixie</td><td><span class="red">3.0.8-2</span></td><td><span class="red">vulnerable</span></td></tr> | ||||
|       <tr><td></td><td>trixie (security)</td><td>3.0.12-1</td><td>fixed</td></tr> | ||||
|     </table> | ||||
|   </body> | ||||
| </html> | ||||
| @@ -0,0 +1,21 @@ | ||||
| <!DOCTYPE html> | ||||
| <html> | ||||
|   <head> | ||||
|     <title>DSA-2024-124</title> | ||||
|   </head> | ||||
|   <body> | ||||
|     <header><h1>DSA-2024-124</h1></header> | ||||
|     <table> | ||||
|       <tr><td><b>Name</b></td><td>DSA-2024-124</td></tr> | ||||
|       <tr><td><b>Description</b></td><td>zlib - security update</td></tr> | ||||
|       <tr><td><b>Source</b></td><td><a href="https://www.debian.org/security/dsa-2024-124">Debian</a></td></tr> | ||||
|       <tr><td><b>References</b></td><td><a href="/tracker/CVE-2024-2000">CVE-2024-2000</a></td></tr> | ||||
|     </table> | ||||
|     <h2>Vulnerable and fixed packages</h2> | ||||
|     <table> | ||||
|       <tr><th>Source Package</th><th>Release</th><th>Version</th><th>Status</th></tr> | ||||
|       <tr><td><a href="/tracker/source-package/zlib">zlib</a></td><td>bookworm</td><td><span class="red">1:1.3.1-1</span></td><td><span class="red">vulnerable</span></td></tr> | ||||
|       <tr><td></td><td>trixie</td><td><span class="red">1:1.3.1-2</span></td><td><span class="red">vulnerable</span></td></tr> | ||||
|     </table> | ||||
|   </body> | ||||
| </html> | ||||
| @@ -0,0 +1,7 @@ | ||||
| [12 Sep 2024] DSA-2024-123 openssl - security update | ||||
| 	{CVE-2024-1000 CVE-2024-1001} | ||||
| 	[bookworm] - openssl 1:1.1.1n-0+deb11u5 | ||||
| 	[trixie] - openssl 3.0.12-1 | ||||
| [10 Sep 2024] DSA-2024-124 zlib - security update | ||||
| 	{CVE-2024-2000} | ||||
| 	[bookworm] - zlib 1:1.3.2-1 | ||||
| @@ -0,0 +1,13 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|   <PropertyGroup> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
| @@ -0,0 +1,3 @@ | ||||
| using System.Runtime.CompilerServices; | ||||
|  | ||||
| [assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.Debian.Tests")] | ||||
| @@ -1,29 +0,0 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian; | ||||
|  | ||||
| public sealed class DistroDebianConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public string Name => "distro-debian"; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) => true; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); | ||||
|  | ||||
|     private sealed class StubConnector : IFeedConnector | ||||
|     { | ||||
|         public StubConnector(string sourceName) => SourceName = sourceName; | ||||
|  | ||||
|         public string SourceName { get; } | ||||
|  | ||||
|         public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|  | ||||
|         public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|  | ||||
|         public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; | ||||
|     } | ||||
| } | ||||
|  | ||||
| @@ -0,0 +1,87 @@ | ||||
| using System; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Configuration; | ||||
|  | ||||
| public sealed class DebianOptions | ||||
| { | ||||
|     public const string HttpClientName = "feedser.debian"; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Raw advisory list published by the Debian security tracker team. | ||||
|     /// Defaults to the Salsa Git raw endpoint to avoid HTML scraping. | ||||
|     /// </summary> | ||||
|     public Uri ListEndpoint { get; set; } = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list"); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Base URI for advisory detail pages. Connector appends {AdvisoryId}. | ||||
|     /// </summary> | ||||
|     public Uri DetailBaseUri { get; set; } = new("https://security-tracker.debian.org/tracker/"); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Maximum advisories fetched per run to cap backfill effort. | ||||
|     /// </summary> | ||||
|     public int MaxAdvisoriesPerFetch { get; set; } = 40; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Initial history window pulled on first run. | ||||
|     /// </summary> | ||||
|     public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Resume overlap to accommodate late edits of existing advisories. | ||||
|     /// </summary> | ||||
|     public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(2); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Request timeout used for list/detail fetches unless overridden via HTTP client. | ||||
|     /// </summary> | ||||
|     public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45); | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Optional pacing delay between detail fetches. | ||||
|     /// </summary> | ||||
|     public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero; | ||||
|  | ||||
|     /// <summary> | ||||
|     /// Custom user-agent for Debian tracker courtesy. | ||||
|     /// </summary> | ||||
|     public string UserAgent { get; set; } = "StellaOps.Feedser.Debian/0.1 (+https://stella-ops.org)"; | ||||
|  | ||||
|     public void Validate() | ||||
|     { | ||||
|         if (ListEndpoint is null || !ListEndpoint.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("Debian list endpoint must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (DetailBaseUri is null || !DetailBaseUri.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("Debian detail base URI must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (MaxAdvisoriesPerFetch <= 0 || MaxAdvisoriesPerFetch > 200) | ||||
|         { | ||||
|             throw new InvalidOperationException("MaxAdvisoriesPerFetch must be between 1 and 200."); | ||||
|         } | ||||
|  | ||||
|         if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365)) | ||||
|         { | ||||
|             throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days."); | ||||
|         } | ||||
|  | ||||
|         if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14)) | ||||
|         { | ||||
|             throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days."); | ||||
|         } | ||||
|  | ||||
|         if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5)) | ||||
|         { | ||||
|             throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes."); | ||||
|         } | ||||
|  | ||||
|         if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10)) | ||||
|         { | ||||
|             throw new InvalidOperationException("RequestDelay must be between 0 and 10 seconds."); | ||||
|         } | ||||
|     } | ||||
| } | ||||
							
								
								
									
										637
									
								
								src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										637
									
								
								src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,637 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Linq; | ||||
| using System.Net; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using MongoDB.Bson; | ||||
| using MongoDB.Bson.IO; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Source.Common; | ||||
| using StellaOps.Feedser.Source.Common.Fetch; | ||||
| using StellaOps.Feedser.Source.Distro.Debian.Configuration; | ||||
| using StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
| using StellaOps.Feedser.Storage.Mongo; | ||||
| using StellaOps.Feedser.Storage.Mongo.Advisories; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
| using StellaOps.Feedser.Storage.Mongo.Dtos; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian; | ||||
|  | ||||
| public sealed class DebianConnector : IFeedConnector | ||||
| { | ||||
|     private const string SchemaVersion = "debian.v1"; | ||||
|  | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly RawDocumentStorage _rawDocumentStorage; | ||||
|     private readonly IDocumentStore _documentStore; | ||||
|     private readonly IDtoStore _dtoStore; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly ISourceStateRepository _stateRepository; | ||||
|     private readonly DebianOptions _options; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<DebianConnector> _logger; | ||||
|  | ||||
|     private static readonly Action<ILogger, string, int, Exception?> LogMapped = | ||||
|         LoggerMessage.Define<string, int>( | ||||
|             LogLevel.Information, | ||||
|             new EventId(1, "DebianMapped"), | ||||
|             "Debian advisory {AdvisoryId} mapped with {AffectedCount} packages"); | ||||
|  | ||||
|     public DebianConnector( | ||||
|         SourceFetchService fetchService, | ||||
|         RawDocumentStorage rawDocumentStorage, | ||||
|         IDocumentStore documentStore, | ||||
|         IDtoStore dtoStore, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         ISourceStateRepository stateRepository, | ||||
|         IOptions<DebianOptions> options, | ||||
|         TimeProvider? timeProvider, | ||||
|         ILogger<DebianConnector> logger) | ||||
|     { | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); | ||||
|         _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); | ||||
|         _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public string SourceName => DebianConnectorPlugin.SourceName; | ||||
|  | ||||
|     public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|  | ||||
|         var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments); | ||||
|         var pendingMappings = new HashSet<Guid>(cursor.PendingMappings); | ||||
|         var fetchCache = new Dictionary<string, DebianFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); | ||||
|         var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         var listUri = _options.ListEndpoint; | ||||
|         var listKey = listUri.ToString(); | ||||
|         touchedResources.Add(listKey); | ||||
|  | ||||
|         var existingList = await _documentStore.FindBySourceAndUriAsync(SourceName, listKey, cancellationToken).ConfigureAwait(false); | ||||
|         cursor.TryGetCache(listKey, out var cachedListEntry); | ||||
|  | ||||
|         var listRequest = new SourceFetchRequest(DebianOptions.HttpClientName, SourceName, listUri) | ||||
|         { | ||||
|             Metadata = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|             { | ||||
|                 ["type"] = "index" | ||||
|             }, | ||||
|             AcceptHeaders = new[] { "text/plain", "text/plain; charset=utf-8" }, | ||||
|             TimeoutOverride = _options.FetchTimeout, | ||||
|             ETag = existingList?.Etag ?? cachedListEntry?.ETag, | ||||
|             LastModified = existingList?.LastModified ?? cachedListEntry?.LastModified, | ||||
|         }; | ||||
|  | ||||
|         SourceFetchResult listResult; | ||||
|         try | ||||
|         { | ||||
|             listResult = await _fetchService.FetchAsync(listRequest, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         catch (Exception ex) | ||||
|         { | ||||
|             _logger.LogError(ex, "Debian list fetch failed"); | ||||
|             await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|             throw; | ||||
|         } | ||||
|  | ||||
|         var lastPublished = cursor.LastPublished ?? (now - _options.InitialBackfill); | ||||
|         var processedIds = new HashSet<string>(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); | ||||
|         var newProcessedIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|         var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; | ||||
|         var processedUpdated = false; | ||||
|  | ||||
|         if (listResult.IsNotModified) | ||||
|         { | ||||
|             if (existingList is not null) | ||||
|             { | ||||
|                 fetchCache[listKey] = DebianFetchCacheEntry.FromDocument(existingList); | ||||
|             } | ||||
|         } | ||||
|         else if (listResult.IsSuccess && listResult.Document is not null) | ||||
|         { | ||||
|             fetchCache[listKey] = DebianFetchCacheEntry.FromDocument(listResult.Document); | ||||
|  | ||||
|             if (!listResult.Document.GridFsId.HasValue) | ||||
|             { | ||||
|                 _logger.LogWarning("Debian list document {DocumentId} missing GridFS payload", listResult.Document.Id); | ||||
|             } | ||||
|             else | ||||
|             { | ||||
|                 byte[] bytes; | ||||
|                 try | ||||
|                 { | ||||
|                     bytes = await _rawDocumentStorage.DownloadAsync(listResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|                 catch (Exception ex) | ||||
|                 { | ||||
|                     _logger.LogError(ex, "Failed to download Debian list document {DocumentId}", listResult.Document.Id); | ||||
|                     throw; | ||||
|                 } | ||||
|  | ||||
|                 var text = System.Text.Encoding.UTF8.GetString(bytes); | ||||
|                 var entries = DebianListParser.Parse(text); | ||||
|                 if (entries.Count > 0) | ||||
|                 { | ||||
|                     var windowStart = (cursor.LastPublished ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap; | ||||
|                     if (windowStart < DateTimeOffset.UnixEpoch) | ||||
|                     { | ||||
|                         windowStart = DateTimeOffset.UnixEpoch; | ||||
|                     } | ||||
|  | ||||
|                     ProvenanceDiagnostics.ReportResumeWindow(SourceName, windowStart, _logger); | ||||
|  | ||||
|                     var candidates = entries | ||||
|                         .Where(entry => entry.Published >= windowStart) | ||||
|                         .OrderBy(entry => entry.Published) | ||||
|                         .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) | ||||
|                         .ToList(); | ||||
|  | ||||
|                     if (candidates.Count == 0) | ||||
|                     { | ||||
|                         candidates = entries | ||||
|                             .OrderByDescending(entry => entry.Published) | ||||
|                             .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) | ||||
|                             .Take(_options.MaxAdvisoriesPerFetch) | ||||
|                             .OrderBy(entry => entry.Published) | ||||
|                             .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) | ||||
|                             .ToList(); | ||||
|                     } | ||||
|                     else if (candidates.Count > _options.MaxAdvisoriesPerFetch) | ||||
|                     { | ||||
|                         candidates = candidates | ||||
|                             .OrderByDescending(entry => entry.Published) | ||||
|                             .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) | ||||
|                             .Take(_options.MaxAdvisoriesPerFetch) | ||||
|                             .OrderBy(entry => entry.Published) | ||||
|                             .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) | ||||
|                             .ToList(); | ||||
|                     } | ||||
|  | ||||
|                     foreach (var entry in candidates) | ||||
|                     { | ||||
|                         cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|                         var detailUri = new Uri(_options.DetailBaseUri, entry.AdvisoryId); | ||||
|                         var cacheKey = detailUri.ToString(); | ||||
|                         touchedResources.Add(cacheKey); | ||||
|  | ||||
|                         cursor.TryGetCache(cacheKey, out var cachedDetail); | ||||
|                         if (!fetchCache.TryGetValue(cacheKey, out var cachedInRun)) | ||||
|                         { | ||||
|                             cachedInRun = cachedDetail; | ||||
|                         } | ||||
|  | ||||
|                         var metadata = BuildDetailMetadata(entry); | ||||
|                         var existingDetail = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|                         var request = new SourceFetchRequest(DebianOptions.HttpClientName, SourceName, detailUri) | ||||
|                         { | ||||
|                             Metadata = metadata, | ||||
|                             AcceptHeaders = new[] { "text/html", "application/xhtml+xml" }, | ||||
|                             TimeoutOverride = _options.FetchTimeout, | ||||
|                             ETag = existingDetail?.Etag ?? cachedInRun?.ETag, | ||||
|                             LastModified = existingDetail?.LastModified ?? cachedInRun?.LastModified, | ||||
|                         }; | ||||
|  | ||||
|                         SourceFetchResult result; | ||||
|                         try | ||||
|                         { | ||||
|                             result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); | ||||
|                         } | ||||
|                         catch (Exception ex) | ||||
|                         { | ||||
|                             _logger.LogError(ex, "Failed to fetch Debian advisory {AdvisoryId}", entry.AdvisoryId); | ||||
|                             await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|                             throw; | ||||
|                         } | ||||
|  | ||||
|                         if (result.IsNotModified) | ||||
|                         { | ||||
|                             if (existingDetail is not null) | ||||
|                             { | ||||
|                                 fetchCache[cacheKey] = DebianFetchCacheEntry.FromDocument(existingDetail); | ||||
|                                 if (string.Equals(existingDetail.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) | ||||
|                                 { | ||||
|                                     pendingDocuments.Remove(existingDetail.Id); | ||||
|                                     pendingMappings.Remove(existingDetail.Id); | ||||
|                                 } | ||||
|                             } | ||||
|  | ||||
|                             continue; | ||||
|                         } | ||||
|  | ||||
|                         if (!result.IsSuccess || result.Document is null) | ||||
|                         { | ||||
|                             continue; | ||||
|                         } | ||||
|  | ||||
|                         fetchCache[cacheKey] = DebianFetchCacheEntry.FromDocument(result.Document); | ||||
|                         pendingDocuments.Add(result.Document.Id); | ||||
|                         pendingMappings.Remove(result.Document.Id); | ||||
|  | ||||
|                         if (_options.RequestDelay > TimeSpan.Zero) | ||||
|                         { | ||||
|                             try | ||||
|                             { | ||||
|                                 await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); | ||||
|                             } | ||||
|                             catch (TaskCanceledException) | ||||
|                             { | ||||
|                                 break; | ||||
|                             } | ||||
|                         } | ||||
|  | ||||
|                         if (entry.Published > maxPublished) | ||||
|                         { | ||||
|                             maxPublished = entry.Published; | ||||
|                             newProcessedIds.Clear(); | ||||
|                             processedUpdated = true; | ||||
|                         } | ||||
|  | ||||
|                         if (entry.Published == maxPublished) | ||||
|                         { | ||||
|                             newProcessedIds.Add(entry.AdvisoryId); | ||||
|                             processedUpdated = true; | ||||
|                         } | ||||
|                     } | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (fetchCache.Count > 0 && touchedResources.Count > 0) | ||||
|         { | ||||
|             var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); | ||||
|             foreach (var key in stale) | ||||
|             { | ||||
|                 fetchCache.Remove(key); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (!processedUpdated && cursor.LastPublished.HasValue) | ||||
|         { | ||||
|             maxPublished = cursor.LastPublished.Value; | ||||
|             newProcessedIds = new HashSet<string>(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(pendingDocuments) | ||||
|             .WithPendingMappings(pendingMappings) | ||||
|             .WithFetchCache(fetchCache); | ||||
|  | ||||
|         if (processedUpdated && maxPublished > DateTimeOffset.MinValue) | ||||
|         { | ||||
|             updatedCursor = updatedCursor.WithProcessed(maxPublished, newProcessedIds); | ||||
|         } | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingDocuments.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var remaining = cursor.PendingDocuments.ToList(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToList(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingDocuments) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 remaining.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!document.GridFsId.HasValue) | ||||
|             { | ||||
|                 _logger.LogWarning("Debian document {DocumentId} missing GridFS payload", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remaining.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var metadata = ExtractMetadata(document); | ||||
|             if (metadata is null) | ||||
|             { | ||||
|                 _logger.LogWarning("Debian document {DocumentId} missing required metadata", document.Id); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remaining.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             byte[] bytes; | ||||
|             try | ||||
|             { | ||||
|                 bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogError(ex, "Failed to download Debian document {DocumentId}", document.Id); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             var html = System.Text.Encoding.UTF8.GetString(bytes); | ||||
|             DebianAdvisoryDto dto; | ||||
|             try | ||||
|             { | ||||
|                 dto = DebianHtmlParser.Parse(html, metadata); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogWarning(ex, "Failed to parse Debian advisory {AdvisoryId}", metadata.AdvisoryId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remaining.Remove(document.Id); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var payload = ToBson(dto); | ||||
|             var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, SchemaVersion, payload, _timeProvider.GetUtcNow()); | ||||
|             await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             remaining.Remove(document.Id); | ||||
|             if (!pendingMappings.Contains(document.Id)) | ||||
|             { | ||||
|                 pendingMappings.Add(document.Id); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(remaining) | ||||
|             .WithPendingMappings(pendingMappings); | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingMappings.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingMappings = cursor.PendingMappings.ToList(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingMappings) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (dtoRecord is null || document is null) | ||||
|             { | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             DebianAdvisoryDto dto; | ||||
|             try | ||||
|             { | ||||
|                 dto = FromBson(dtoRecord.Payload); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogError(ex, "Failed to deserialize Debian DTO for document {DocumentId}", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var advisory = DebianMapper.Map(dto, document, _timeProvider.GetUtcNow()); | ||||
|             await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|             pendingMappings.Remove(documentId); | ||||
|             LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingMappings(pendingMappings); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private async Task<DebianCursor> GetCursorAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); | ||||
|         return state is null ? DebianCursor.Empty : DebianCursor.FromBson(state.Cursor); | ||||
|     } | ||||
|  | ||||
|     private async Task UpdateCursorAsync(DebianCursor cursor, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var document = cursor.ToBsonDocument(); | ||||
|         await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private static Dictionary<string, string> BuildDetailMetadata(DebianListEntry entry) | ||||
|     { | ||||
|         var metadata = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|         { | ||||
|             ["debian.id"] = entry.AdvisoryId, | ||||
|             ["debian.published"] = entry.Published.ToString("O", CultureInfo.InvariantCulture), | ||||
|             ["debian.title"] = entry.Title, | ||||
|             ["debian.package"] = entry.SourcePackage | ||||
|         }; | ||||
|  | ||||
|         if (entry.CveIds.Count > 0) | ||||
|         { | ||||
|             metadata["debian.cves"] = string.Join(' ', entry.CveIds); | ||||
|         } | ||||
|  | ||||
|         return metadata; | ||||
|     } | ||||
|  | ||||
|     private static DebianDetailMetadata? ExtractMetadata(DocumentRecord document) | ||||
|     { | ||||
|         if (document.Metadata is null) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (!document.Metadata.TryGetValue("debian.id", out var id) || string.IsNullOrWhiteSpace(id)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (!document.Metadata.TryGetValue("debian.published", out var publishedRaw) | ||||
|             || !DateTimeOffset.TryParse(publishedRaw, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published)) | ||||
|         { | ||||
|             published = document.FetchedAt; | ||||
|         } | ||||
|  | ||||
|         var title = document.Metadata.TryGetValue("debian.title", out var t) ? t : id; | ||||
|         var package = document.Metadata.TryGetValue("debian.package", out var pkg) && !string.IsNullOrWhiteSpace(pkg) | ||||
|             ? pkg | ||||
|             : id; | ||||
|  | ||||
|         IReadOnlyList<string> cveList = Array.Empty<string>(); | ||||
|         if (document.Metadata.TryGetValue("debian.cves", out var cvesRaw) && !string.IsNullOrWhiteSpace(cvesRaw)) | ||||
|         { | ||||
|             cveList = cvesRaw | ||||
|                 .Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) | ||||
|                 .Where(static s => !string.IsNullOrWhiteSpace(s)) | ||||
|                 .Select(static s => s!) | ||||
|                 .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|                 .ToArray(); | ||||
|         } | ||||
|  | ||||
|         return new DebianDetailMetadata( | ||||
|             id.Trim(), | ||||
|             new Uri(document.Uri, UriKind.Absolute), | ||||
|             published.ToUniversalTime(), | ||||
|             title, | ||||
|             package, | ||||
|             cveList); | ||||
|     } | ||||
|  | ||||
|     private static BsonDocument ToBson(DebianAdvisoryDto dto) | ||||
|     { | ||||
|         var packages = new BsonArray(); | ||||
|         foreach (var package in dto.Packages) | ||||
|         { | ||||
|             var packageDoc = new BsonDocument | ||||
|             { | ||||
|                 ["package"] = package.Package, | ||||
|                 ["release"] = package.Release, | ||||
|                 ["status"] = package.Status, | ||||
|             }; | ||||
|  | ||||
|             if (!string.IsNullOrWhiteSpace(package.IntroducedVersion)) | ||||
|             { | ||||
|                 packageDoc["introduced"] = package.IntroducedVersion; | ||||
|             } | ||||
|  | ||||
|             if (!string.IsNullOrWhiteSpace(package.FixedVersion)) | ||||
|             { | ||||
|                 packageDoc["fixed"] = package.FixedVersion; | ||||
|             } | ||||
|  | ||||
|             if (!string.IsNullOrWhiteSpace(package.LastAffectedVersion)) | ||||
|             { | ||||
|                 packageDoc["last"] = package.LastAffectedVersion; | ||||
|             } | ||||
|  | ||||
|             if (package.Published.HasValue) | ||||
|             { | ||||
|                 packageDoc["published"] = package.Published.Value.UtcDateTime; | ||||
|             } | ||||
|  | ||||
|             packages.Add(packageDoc); | ||||
|         } | ||||
|  | ||||
|         var references = new BsonArray(dto.References.Select(reference => | ||||
|         { | ||||
|             var doc = new BsonDocument | ||||
|             { | ||||
|                 ["url"] = reference.Url | ||||
|             }; | ||||
|  | ||||
|             if (!string.IsNullOrWhiteSpace(reference.Kind)) | ||||
|             { | ||||
|                 doc["kind"] = reference.Kind; | ||||
|             } | ||||
|  | ||||
|             if (!string.IsNullOrWhiteSpace(reference.Title)) | ||||
|             { | ||||
|                 doc["title"] = reference.Title; | ||||
|             } | ||||
|  | ||||
|             return doc; | ||||
|         })); | ||||
|  | ||||
|         return new BsonDocument | ||||
|         { | ||||
|             ["advisoryId"] = dto.AdvisoryId, | ||||
|             ["sourcePackage"] = dto.SourcePackage, | ||||
|             ["title"] = dto.Title, | ||||
|             ["description"] = dto.Description ?? string.Empty, | ||||
|             ["cves"] = new BsonArray(dto.CveIds), | ||||
|             ["packages"] = packages, | ||||
|             ["references"] = references, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static DebianAdvisoryDto FromBson(BsonDocument document) | ||||
|     { | ||||
|         var advisoryId = document.GetValue("advisoryId", "").AsString; | ||||
|         var sourcePackage = document.GetValue("sourcePackage", advisoryId).AsString; | ||||
|         var title = document.GetValue("title", advisoryId).AsString; | ||||
|         var description = document.TryGetValue("description", out var desc) ? desc.AsString : null; | ||||
|  | ||||
|         var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray cvesBson | ||||
|             ? cvesBson.OfType<BsonValue>() | ||||
|                 .Select(static value => value.ToString()) | ||||
|                 .Where(static s => !string.IsNullOrWhiteSpace(s)) | ||||
|                 .Select(static s => s!) | ||||
|                 .ToArray() | ||||
|             : Array.Empty<string>(); | ||||
|  | ||||
|         var packages = new List<DebianPackageStateDto>(); | ||||
|         if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray packagesBson) | ||||
|         { | ||||
|             foreach (var element in packagesBson.OfType<BsonDocument>()) | ||||
|             { | ||||
|                 packages.Add(new DebianPackageStateDto( | ||||
|                     element.GetValue("package", sourcePackage).AsString, | ||||
|                     element.GetValue("release", string.Empty).AsString, | ||||
|                     element.GetValue("status", "unknown").AsString, | ||||
|                     element.TryGetValue("introduced", out var introducedValue) ? introducedValue.AsString : null, | ||||
|                     element.TryGetValue("fixed", out var fixedValue) ? fixedValue.AsString : null, | ||||
|                     element.TryGetValue("last", out var lastValue) ? lastValue.AsString : null, | ||||
|                     element.TryGetValue("published", out var publishedValue) | ||||
|                         ? publishedValue.BsonType switch | ||||
|                         { | ||||
|                             BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc), | ||||
|                             BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|                             _ => (DateTimeOffset?)null, | ||||
|                         } | ||||
|                         : null)); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var references = new List<DebianReferenceDto>(); | ||||
|         if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray refBson) | ||||
|         { | ||||
|             foreach (var element in refBson.OfType<BsonDocument>()) | ||||
|             { | ||||
|                 references.Add(new DebianReferenceDto( | ||||
|                     element.GetValue("url", "").AsString, | ||||
|                     element.TryGetValue("kind", out var kind) ? kind.AsString : null, | ||||
|                     element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null)); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return new DebianAdvisoryDto( | ||||
|             advisoryId, | ||||
|             sourcePackage, | ||||
|             title, | ||||
|             description, | ||||
|             cves, | ||||
|             packages, | ||||
|             references); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,22 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian; | ||||
|  | ||||
| public sealed class DebianConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public const string SourceName = "distro-debian"; | ||||
|  | ||||
|     public string Name => SourceName; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) => services is not null; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return ActivatorUtilities.CreateInstance<DebianConnector>(services); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,53 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.DependencyInjection; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
| using StellaOps.Feedser.Source.Distro.Debian.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian; | ||||
|  | ||||
| public sealed class DebianDependencyInjectionRoutine : IDependencyInjectionRoutine | ||||
| { | ||||
|     private const string ConfigurationSection = "feedser:sources:debian"; | ||||
|     private const string FetchSchedule = "*/30 * * * *"; | ||||
|     private const string ParseSchedule = "7,37 * * * *"; | ||||
|     private const string MapSchedule = "12,42 * * * *"; | ||||
|  | ||||
|     private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6); | ||||
|     private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10); | ||||
|     private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(10); | ||||
|     private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5); | ||||
|  | ||||
|     public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         services.AddDebianConnector(options => | ||||
|         { | ||||
|             configuration.GetSection(ConfigurationSection).Bind(options); | ||||
|             options.Validate(); | ||||
|         }); | ||||
|  | ||||
|         var scheduler = new JobSchedulerBuilder(services); | ||||
|         scheduler | ||||
|             .AddJob<DebianFetchJob>( | ||||
|                 DebianJobKinds.Fetch, | ||||
|                 cronExpression: FetchSchedule, | ||||
|                 timeout: FetchTimeout, | ||||
|                 leaseDuration: LeaseDuration) | ||||
|             .AddJob<DebianParseJob>( | ||||
|                 DebianJobKinds.Parse, | ||||
|                 cronExpression: ParseSchedule, | ||||
|                 timeout: ParseTimeout, | ||||
|                 leaseDuration: LeaseDuration) | ||||
|             .AddJob<DebianMapJob>( | ||||
|                 DebianJobKinds.Map, | ||||
|                 cronExpression: MapSchedule, | ||||
|                 timeout: MapTimeout, | ||||
|                 leaseDuration: LeaseDuration); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,37 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Feedser.Source.Common.Http; | ||||
| using StellaOps.Feedser.Source.Distro.Debian.Configuration; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian; | ||||
|  | ||||
| public static class DebianServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddDebianConnector(this IServiceCollection services, Action<DebianOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|  | ||||
|         services.AddOptions<DebianOptions>() | ||||
|             .Configure(configure) | ||||
|             .PostConfigure(static options => options.Validate()); | ||||
|  | ||||
|         services.AddSourceHttpClient(DebianOptions.HttpClientName, (sp, httpOptions) => | ||||
|         { | ||||
|             var options = sp.GetRequiredService<IOptions<DebianOptions>>().Value; | ||||
|             httpOptions.BaseAddress = options.DetailBaseUri.GetLeftPart(UriPartial.Authority) is { Length: > 0 } authority | ||||
|                 ? new Uri(authority, UriKind.Absolute) | ||||
|                 : new Uri("https://security-tracker.debian.org/", UriKind.Absolute); | ||||
|             httpOptions.Timeout = options.FetchTimeout; | ||||
|             httpOptions.UserAgent = options.UserAgent; | ||||
|             httpOptions.AllowedHosts.Clear(); | ||||
|             httpOptions.AllowedHosts.Add(options.DetailBaseUri.Host); | ||||
|             httpOptions.AllowedHosts.Add(options.ListEndpoint.Host); | ||||
|             httpOptions.DefaultRequestHeaders["Accept"] = "text/html,application/xhtml+xml,text/plain;q=0.9,application/json;q=0.8"; | ||||
|         }); | ||||
|  | ||||
|         services.AddTransient<DebianConnector>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,27 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
|  | ||||
| internal sealed record DebianAdvisoryDto( | ||||
|     string AdvisoryId, | ||||
|     string SourcePackage, | ||||
|     string? Title, | ||||
|     string? Description, | ||||
|     IReadOnlyList<string> CveIds, | ||||
|     IReadOnlyList<DebianPackageStateDto> Packages, | ||||
|     IReadOnlyList<DebianReferenceDto> References); | ||||
|  | ||||
| internal sealed record DebianPackageStateDto( | ||||
|     string Package, | ||||
|     string Release, | ||||
|     string Status, | ||||
|     string? IntroducedVersion, | ||||
|     string? FixedVersion, | ||||
|     string? LastAffectedVersion, | ||||
|     DateTimeOffset? Published); | ||||
|  | ||||
| internal sealed record DebianReferenceDto( | ||||
|     string Url, | ||||
|     string? Kind, | ||||
|     string? Title); | ||||
| @@ -0,0 +1,177 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
|  | ||||
| internal sealed record DebianCursor( | ||||
|     DateTimeOffset? LastPublished, | ||||
|     IReadOnlyCollection<string> ProcessedAdvisoryIds, | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings, | ||||
|     IReadOnlyDictionary<string, DebianFetchCacheEntry> FetchCache) | ||||
| { | ||||
|     private static readonly IReadOnlyCollection<string> EmptyIds = Array.Empty<string>(); | ||||
|     private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); | ||||
|     private static readonly IReadOnlyDictionary<string, DebianFetchCacheEntry> EmptyCache = | ||||
|         new Dictionary<string, DebianFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|     public static DebianCursor Empty { get; } = new(null, EmptyIds, EmptyGuidList, EmptyGuidList, EmptyCache); | ||||
|  | ||||
|     public static DebianCursor FromBson(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         DateTimeOffset? lastPublished = null; | ||||
|         if (document.TryGetValue("lastPublished", out var lastValue)) | ||||
|         { | ||||
|             lastPublished = lastValue.BsonType switch | ||||
|             { | ||||
|                 BsonType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|                 BsonType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc), | ||||
|                 _ => null, | ||||
|             }; | ||||
|         } | ||||
|  | ||||
|         var processed = ReadStringArray(document, "processedIds"); | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|         var cache = ReadCache(document); | ||||
|  | ||||
|         return new DebianCursor(lastPublished, processed, pendingDocuments, pendingMappings, cache); | ||||
|     } | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), | ||||
|             ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), | ||||
|         }; | ||||
|  | ||||
|         if (LastPublished.HasValue) | ||||
|         { | ||||
|             document["lastPublished"] = LastPublished.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         if (ProcessedAdvisoryIds.Count > 0) | ||||
|         { | ||||
|             document["processedIds"] = new BsonArray(ProcessedAdvisoryIds); | ||||
|         } | ||||
|  | ||||
|         if (FetchCache.Count > 0) | ||||
|         { | ||||
|             var cacheDoc = new BsonDocument(); | ||||
|             foreach (var (key, entry) in FetchCache) | ||||
|             { | ||||
|                 cacheDoc[key] = entry.ToBsonDocument(); | ||||
|             } | ||||
|  | ||||
|             document["fetchCache"] = cacheDoc; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public DebianCursor WithPendingDocuments(IEnumerable<Guid> ids) | ||||
|         => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; | ||||
|  | ||||
|     public DebianCursor WithPendingMappings(IEnumerable<Guid> ids) | ||||
|         => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; | ||||
|  | ||||
|     public DebianCursor WithProcessed(DateTimeOffset published, IEnumerable<string> ids) | ||||
|         => this with | ||||
|         { | ||||
|             LastPublished = published.ToUniversalTime(), | ||||
|             ProcessedAdvisoryIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id)) | ||||
|                 .Select(static id => id.Trim()) | ||||
|                 .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|                 .ToArray() ?? EmptyIds | ||||
|         }; | ||||
|  | ||||
|     public DebianCursor WithFetchCache(IDictionary<string, DebianFetchCacheEntry>? cache) | ||||
|     { | ||||
|         if (cache is null || cache.Count == 0) | ||||
|         { | ||||
|             return this with { FetchCache = EmptyCache }; | ||||
|         } | ||||
|  | ||||
|         return this with { FetchCache = new Dictionary<string, DebianFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) }; | ||||
|     } | ||||
|  | ||||
|     public bool TryGetCache(string key, out DebianFetchCacheEntry entry) | ||||
|     { | ||||
|         if (FetchCache.Count == 0) | ||||
|         { | ||||
|             entry = DebianFetchCacheEntry.Empty; | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         return FetchCache.TryGetValue(key, out entry!); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyIds; | ||||
|         } | ||||
|  | ||||
|         var list = new List<string>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (element.BsonType == BsonType.String) | ||||
|             { | ||||
|                 var str = element.AsString.Trim(); | ||||
|                 if (!string.IsNullOrEmpty(str)) | ||||
|                 { | ||||
|                     list.Add(str); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyGuidList; | ||||
|         } | ||||
|  | ||||
|         var list = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (Guid.TryParse(element.ToString(), out var guid)) | ||||
|             { | ||||
|                 list.Add(guid); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, DebianFetchCacheEntry> ReadCache(BsonDocument document) | ||||
|     { | ||||
|         if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) | ||||
|         { | ||||
|             return EmptyCache; | ||||
|         } | ||||
|  | ||||
|         var cache = new Dictionary<string, DebianFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); | ||||
|         foreach (var element in cacheDocument.Elements) | ||||
|         { | ||||
|             if (element.Value is BsonDocument entry) | ||||
|             { | ||||
|                 cache[element.Name] = DebianFetchCacheEntry.FromBson(entry); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return cache; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,12 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
|  | ||||
| internal sealed record DebianDetailMetadata( | ||||
|     string AdvisoryId, | ||||
|     Uri DetailUri, | ||||
|     DateTimeOffset Published, | ||||
|     string Title, | ||||
|     string SourcePackage, | ||||
|     IReadOnlyList<string> CveIds); | ||||
| @@ -0,0 +1,76 @@ | ||||
| using System; | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
|  | ||||
| internal sealed record DebianFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) | ||||
| { | ||||
|     public static DebianFetchCacheEntry Empty { get; } = new(null, null); | ||||
|  | ||||
|     public static DebianFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) | ||||
|         => new(document.Etag, document.LastModified); | ||||
|  | ||||
|     public static DebianFetchCacheEntry FromBson(BsonDocument document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         string? etag = null; | ||||
|         DateTimeOffset? lastModified = null; | ||||
|  | ||||
|         if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String) | ||||
|         { | ||||
|             etag = etagValue.AsString; | ||||
|         } | ||||
|  | ||||
|         if (document.TryGetValue("lastModified", out var modifiedValue)) | ||||
|         { | ||||
|             lastModified = modifiedValue.BsonType switch | ||||
|             { | ||||
|                 BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|                 BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc), | ||||
|                 _ => null, | ||||
|             }; | ||||
|         } | ||||
|  | ||||
|         return new DebianFetchCacheEntry(etag, lastModified); | ||||
|     } | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument(); | ||||
|         if (!string.IsNullOrWhiteSpace(ETag)) | ||||
|         { | ||||
|             document["etag"] = ETag; | ||||
|         } | ||||
|  | ||||
|         if (LastModified.HasValue) | ||||
|         { | ||||
|             document["lastModified"] = LastModified.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) | ||||
|     { | ||||
|         if (document is null) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (!string.Equals(document.Etag, ETag, StringComparison.Ordinal)) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (LastModified.HasValue && document.LastModified.HasValue) | ||||
|         { | ||||
|             return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         return !LastModified.HasValue && !document.LastModified.HasValue; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,326 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Linq; | ||||
| using AngleSharp.Html.Dom; | ||||
| using AngleSharp.Html.Parser; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
|  | ||||
| internal static class DebianHtmlParser | ||||
| { | ||||
|     public static DebianAdvisoryDto Parse(string html, DebianDetailMetadata metadata) | ||||
|     { | ||||
|         ArgumentException.ThrowIfNullOrEmpty(html); | ||||
|         ArgumentNullException.ThrowIfNull(metadata); | ||||
|  | ||||
|         var parser = new HtmlParser(); | ||||
|         var document = parser.ParseDocument(html); | ||||
|  | ||||
|         var description = ExtractDescription(document) ?? metadata.Title; | ||||
|         var references = ExtractReferences(document, metadata); | ||||
|         var packages = ExtractPackages(document, metadata.SourcePackage, metadata.Published); | ||||
|  | ||||
|         return new DebianAdvisoryDto( | ||||
|             metadata.AdvisoryId, | ||||
|             metadata.SourcePackage, | ||||
|             metadata.Title, | ||||
|             description, | ||||
|             metadata.CveIds, | ||||
|             packages, | ||||
|             references); | ||||
|     } | ||||
|  | ||||
|     private static string? ExtractDescription(IHtmlDocument document) | ||||
|     { | ||||
|         foreach (var table in document.QuerySelectorAll("table")) | ||||
|         { | ||||
|             if (table is not IHtmlTableElement tableElement) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             foreach (var row in tableElement.Rows) | ||||
|             { | ||||
|                 if (row.Cells.Length < 2) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var header = row.Cells[0].TextContent?.Trim(); | ||||
|                 if (string.Equals(header, "Description", StringComparison.OrdinalIgnoreCase)) | ||||
|                 { | ||||
|                     return NormalizeWhitespace(row.Cells[1].TextContent); | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             // Only the first table contains the metadata rows we need. | ||||
|             break; | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<DebianReferenceDto> ExtractReferences(IHtmlDocument document, DebianDetailMetadata metadata) | ||||
|     { | ||||
|         var references = new List<DebianReferenceDto>(); | ||||
|         var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|  | ||||
|         // Add canonical Debian advisory page. | ||||
|         var canonical = new Uri($"https://www.debian.org/security/{metadata.AdvisoryId.ToLowerInvariant()}"); | ||||
|         references.Add(new DebianReferenceDto(canonical.ToString(), "advisory", metadata.Title)); | ||||
|         seen.Add(canonical.ToString()); | ||||
|  | ||||
|         foreach (var link in document.QuerySelectorAll("a")) | ||||
|         { | ||||
|             var href = link.GetAttribute("href"); | ||||
|             if (string.IsNullOrWhiteSpace(href)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             string resolved; | ||||
|             if (Uri.TryCreate(href, UriKind.Absolute, out var absolute)) | ||||
|             { | ||||
|                 resolved = absolute.ToString(); | ||||
|             } | ||||
|             else if (Uri.TryCreate(metadata.DetailUri, href, out var relative)) | ||||
|             { | ||||
|                 resolved = relative.ToString(); | ||||
|             } | ||||
|             else | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!seen.Add(resolved)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var text = NormalizeWhitespace(link.TextContent); | ||||
|             string? kind = null; | ||||
|             if (text.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 kind = "cve"; | ||||
|             } | ||||
|             else if (resolved.Contains("debian.org/security", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 kind = "advisory"; | ||||
|             } | ||||
|  | ||||
|             references.Add(new DebianReferenceDto(resolved, kind, text)); | ||||
|         } | ||||
|  | ||||
|         return references; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<DebianPackageStateDto> ExtractPackages(IHtmlDocument document, string defaultPackage, DateTimeOffset published) | ||||
|     { | ||||
|         var table = FindPackagesTable(document); | ||||
|         if (table is null) | ||||
|         { | ||||
|             return Array.Empty<DebianPackageStateDto>(); | ||||
|         } | ||||
|  | ||||
|         var accumulators = new Dictionary<string, PackageAccumulator>(StringComparer.OrdinalIgnoreCase); | ||||
|         string currentPackage = defaultPackage; | ||||
|  | ||||
|         foreach (var body in table.Bodies) | ||||
|         { | ||||
|             foreach (var row in body.Rows) | ||||
|             { | ||||
|                 if (row.Cells.Length < 4) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var packageCell = NormalizeWhitespace(row.Cells[0].TextContent); | ||||
|                 if (!string.IsNullOrWhiteSpace(packageCell)) | ||||
|                 { | ||||
|                     currentPackage = ExtractPackageName(packageCell); | ||||
|                 } | ||||
|  | ||||
|                 if (string.IsNullOrWhiteSpace(currentPackage)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var releaseRaw = NormalizeWhitespace(row.Cells[1].TextContent); | ||||
|                 var versionRaw = NormalizeWhitespace(row.Cells[2].TextContent); | ||||
|                 var statusRaw = NormalizeWhitespace(row.Cells[3].TextContent); | ||||
|                 if (string.IsNullOrWhiteSpace(releaseRaw)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var release = NormalizeRelease(releaseRaw); | ||||
|                 var key = $"{currentPackage}|{release}"; | ||||
|                 if (!accumulators.TryGetValue(key, out var accumulator)) | ||||
|                 { | ||||
|                     accumulator = new PackageAccumulator(currentPackage, release, published); | ||||
|                     accumulators[key] = accumulator; | ||||
|                 } | ||||
|  | ||||
|                 accumulator.Apply(statusRaw, versionRaw); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return accumulators.Values | ||||
|             .Where(static acc => acc.ShouldEmit) | ||||
|             .Select(static acc => acc.ToDto()) | ||||
|             .OrderBy(static dto => dto.Release, StringComparer.OrdinalIgnoreCase) | ||||
|             .ThenBy(static dto => dto.Package, StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static IHtmlTableElement? FindPackagesTable(IHtmlDocument document) | ||||
|     { | ||||
|         foreach (var table in document.QuerySelectorAll("table")) | ||||
|         { | ||||
|             if (table is not IHtmlTableElement tableElement) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var header = tableElement.Rows.FirstOrDefault(); | ||||
|             if (header is null || header.Cells.Length < 4) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var firstHeader = NormalizeWhitespace(header.Cells[0].TextContent); | ||||
|             var secondHeader = NormalizeWhitespace(header.Cells[1].TextContent); | ||||
|             var thirdHeader = NormalizeWhitespace(header.Cells[2].TextContent); | ||||
|             if (string.Equals(firstHeader, "Source Package", StringComparison.OrdinalIgnoreCase) | ||||
|                 && string.Equals(secondHeader, "Release", StringComparison.OrdinalIgnoreCase) | ||||
|                 && string.Equals(thirdHeader, "Version", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 return tableElement; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private static string NormalizeRelease(string release) | ||||
|     { | ||||
|         var trimmed = release.Trim(); | ||||
|         var parenthesisIndex = trimmed.IndexOf('('); | ||||
|         if (parenthesisIndex > 0) | ||||
|         { | ||||
|             trimmed = trimmed[..parenthesisIndex].Trim(); | ||||
|         } | ||||
|  | ||||
|         return trimmed; | ||||
|     } | ||||
|  | ||||
|     private static string ExtractPackageName(string value) | ||||
|     { | ||||
|         var trimmed = value.Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); | ||||
|         if (string.IsNullOrWhiteSpace(trimmed)) | ||||
|         { | ||||
|             return value.Trim(); | ||||
|         } | ||||
|  | ||||
|         if (trimmed.EndsWith(")", StringComparison.Ordinal) && trimmed.Contains('(')) | ||||
|         { | ||||
|             trimmed = trimmed[..trimmed.IndexOf('(')]; | ||||
|         } | ||||
|  | ||||
|         return trimmed.Trim(); | ||||
|     } | ||||
|  | ||||
|     private static string NormalizeWhitespace(string value) | ||||
|         => string.IsNullOrWhiteSpace(value) | ||||
|             ? string.Empty | ||||
|             : string.Join(' ', value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); | ||||
|  | ||||
|     private sealed class PackageAccumulator | ||||
|     { | ||||
|         private readonly DateTimeOffset _published; | ||||
|  | ||||
|         public PackageAccumulator(string package, string release, DateTimeOffset published) | ||||
|         { | ||||
|             Package = package; | ||||
|             Release = release; | ||||
|             _published = published; | ||||
|             Status = "unknown"; | ||||
|         } | ||||
|  | ||||
|         public string Package { get; } | ||||
|  | ||||
|         public string Release { get; } | ||||
|  | ||||
|         public string Status { get; private set; } | ||||
|  | ||||
|         public string? IntroducedVersion { get; private set; } | ||||
|  | ||||
|         public string? FixedVersion { get; private set; } | ||||
|  | ||||
|         public string? LastAffectedVersion { get; private set; } | ||||
|  | ||||
|         public bool ShouldEmit => | ||||
|             !string.Equals(Status, "not_affected", StringComparison.OrdinalIgnoreCase) | ||||
|             || IntroducedVersion is not null | ||||
|             || FixedVersion is not null; | ||||
|  | ||||
|         public void Apply(string statusRaw, string versionRaw) | ||||
|         { | ||||
|             var status = statusRaw.ToLowerInvariant(); | ||||
|             var version = string.IsNullOrWhiteSpace(versionRaw) ? null : versionRaw.Trim(); | ||||
|  | ||||
|             if (status.Contains("fixed", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 FixedVersion = version; | ||||
|                 if (!string.Equals(Status, "open", StringComparison.OrdinalIgnoreCase)) | ||||
|                 { | ||||
|                     Status = "resolved"; | ||||
|                 } | ||||
|  | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             if (status.Contains("vulnerable", StringComparison.OrdinalIgnoreCase) | ||||
|                 || status.Contains("open", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 IntroducedVersion ??= version; | ||||
|                 if (!string.Equals(Status, "resolved", StringComparison.OrdinalIgnoreCase)) | ||||
|                 { | ||||
|                     Status = "open"; | ||||
|                 } | ||||
|  | ||||
|                 LastAffectedVersion = null; | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             if (status.Contains("not affected", StringComparison.OrdinalIgnoreCase) | ||||
|                 || status.Contains("not vulnerable", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 Status = "not_affected"; | ||||
|                 IntroducedVersion = null; | ||||
|                 FixedVersion = null; | ||||
|                 LastAffectedVersion = null; | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             if (status.Contains("end-of-life", StringComparison.OrdinalIgnoreCase) || status.Contains("end of life", StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 Status = "end_of_life"; | ||||
|                 return; | ||||
|             } | ||||
|  | ||||
|             Status = statusRaw; | ||||
|         } | ||||
|  | ||||
|         public DebianPackageStateDto ToDto() | ||||
|             => new( | ||||
|                 Package: Package, | ||||
|                 Release: Release, | ||||
|                 Status: Status, | ||||
|                 IntroducedVersion: IntroducedVersion, | ||||
|                 FixedVersion: FixedVersion, | ||||
|                 LastAffectedVersion: LastAffectedVersion, | ||||
|                 Published: _published); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,11 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
|  | ||||
| internal sealed record DebianListEntry( | ||||
|     string AdvisoryId, | ||||
|     DateTimeOffset Published, | ||||
|     string Title, | ||||
|     string SourcePackage, | ||||
|     IReadOnlyList<string> CveIds); | ||||
| @@ -0,0 +1,107 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Text.RegularExpressions; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
|  | ||||
| internal static class DebianListParser | ||||
| { | ||||
|     private static readonly Regex HeaderRegex = new("^\\[(?<date>[^\\]]+)\\]\\s+(?<id>DSA-\\d{4,}-\\d+)\\s+(?<title>.+)$", RegexOptions.Compiled); | ||||
|     private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); | ||||
|  | ||||
|     public static IReadOnlyList<DebianListEntry> Parse(string? content) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(content)) | ||||
|         { | ||||
|             return Array.Empty<DebianListEntry>(); | ||||
|         } | ||||
|  | ||||
|         var entries = new List<DebianListEntry>(); | ||||
|         var currentCves = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|         DateTimeOffset currentDate = default; | ||||
|         string? currentId = null; | ||||
|         string? currentTitle = null; | ||||
|         string? currentPackage = null; | ||||
|  | ||||
|         foreach (var rawLine in content.Split('\n')) | ||||
|         { | ||||
|             var line = rawLine.TrimEnd('\r'); | ||||
|             if (string.IsNullOrWhiteSpace(line)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (line[0] == '[') | ||||
|             { | ||||
|                 if (currentId is not null && currentTitle is not null && currentPackage is not null) | ||||
|                 { | ||||
|                     entries.Add(new DebianListEntry( | ||||
|                         currentId, | ||||
|                         currentDate, | ||||
|                         currentTitle, | ||||
|                         currentPackage, | ||||
|                         currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves))); | ||||
|                 } | ||||
|  | ||||
|                 currentCves.Clear(); | ||||
|                 currentId = null; | ||||
|                 currentTitle = null; | ||||
|                 currentPackage = null; | ||||
|  | ||||
|                 var match = HeaderRegex.Match(line); | ||||
|                 if (!match.Success) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!DateTimeOffset.TryParseExact( | ||||
|                         match.Groups["date"].Value, | ||||
|                         new[] { "dd MMM yyyy", "d MMM yyyy" }, | ||||
|                         CultureInfo.InvariantCulture, | ||||
|                         DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, | ||||
|                         out currentDate)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 currentId = match.Groups["id"].Value.Trim(); | ||||
|                 currentTitle = match.Groups["title"].Value.Trim(); | ||||
|  | ||||
|                 var separatorIndex = currentTitle.IndexOf(" - ", StringComparison.Ordinal); | ||||
|                 currentPackage = separatorIndex > 0 | ||||
|                     ? currentTitle[..separatorIndex].Trim() | ||||
|                     : currentTitle.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault(); | ||||
|                 if (string.IsNullOrWhiteSpace(currentPackage)) | ||||
|                 { | ||||
|                     currentPackage = currentId; | ||||
|                 } | ||||
|  | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (line[0] == '{') | ||||
|             { | ||||
|                 foreach (Match match in CveRegex.Matches(line)) | ||||
|                 { | ||||
|                     if (match.Success && !string.IsNullOrWhiteSpace(match.Value)) | ||||
|                     { | ||||
|                         currentCves.Add(match.Value.ToUpperInvariant()); | ||||
|                     } | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         if (currentId is not null && currentTitle is not null && currentPackage is not null) | ||||
|         { | ||||
|             entries.Add(new DebianListEntry( | ||||
|                 currentId, | ||||
|                 currentDate, | ||||
|                 currentTitle, | ||||
|                 currentPackage, | ||||
|                 currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves))); | ||||
|         } | ||||
|  | ||||
|         return entries; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,266 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using StellaOps.Feedser.Models; | ||||
| using StellaOps.Feedser.Normalization.Distro; | ||||
| using StellaOps.Feedser.Source.Common; | ||||
| using StellaOps.Feedser.Storage.Mongo.Documents; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian.Internal; | ||||
|  | ||||
| internal static class DebianMapper | ||||
| { | ||||
|     public static Advisory Map( | ||||
|         DebianAdvisoryDto dto, | ||||
|         DocumentRecord document, | ||||
|         DateTimeOffset recordedAt) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(dto); | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|  | ||||
|         var aliases = BuildAliases(dto); | ||||
|         var references = BuildReferences(dto, recordedAt); | ||||
|         var affectedPackages = BuildAffectedPackages(dto, recordedAt); | ||||
|  | ||||
|         var fetchProvenance = new AdvisoryProvenance( | ||||
|             DebianConnectorPlugin.SourceName, | ||||
|             "document", | ||||
|             document.Uri, | ||||
|             document.FetchedAt.ToUniversalTime()); | ||||
|  | ||||
|         var mappingProvenance = new AdvisoryProvenance( | ||||
|             DebianConnectorPlugin.SourceName, | ||||
|             "mapping", | ||||
|             dto.AdvisoryId, | ||||
|             recordedAt); | ||||
|  | ||||
|         return new Advisory( | ||||
|             advisoryKey: dto.AdvisoryId, | ||||
|             title: dto.Title ?? dto.AdvisoryId, | ||||
|             summary: dto.Description, | ||||
|             language: "en", | ||||
|             published: dto.Packages.Select(p => p.Published).Where(p => p.HasValue).Select(p => p!.Value).Cast<DateTimeOffset?>().DefaultIfEmpty(null).Min(), | ||||
|             modified: recordedAt, | ||||
|             severity: null, | ||||
|             exploitKnown: false, | ||||
|             aliases: aliases, | ||||
|             references: references, | ||||
|             affectedPackages: affectedPackages, | ||||
|             cvssMetrics: Array.Empty<CvssMetric>(), | ||||
|             provenance: new[] { fetchProvenance, mappingProvenance }); | ||||
|     } | ||||
|  | ||||
|     private static string[] BuildAliases(DebianAdvisoryDto dto) | ||||
|     { | ||||
|         var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase); | ||||
|         if (!string.IsNullOrWhiteSpace(dto.AdvisoryId)) | ||||
|         { | ||||
|             aliases.Add(dto.AdvisoryId.Trim()); | ||||
|         } | ||||
|  | ||||
|         foreach (var cve in dto.CveIds ?? Array.Empty<string>()) | ||||
|         { | ||||
|             if (!string.IsNullOrWhiteSpace(cve)) | ||||
|             { | ||||
|                 aliases.Add(cve.Trim()); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return aliases.OrderBy(a => a, StringComparer.OrdinalIgnoreCase).ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static AdvisoryReference[] BuildReferences(DebianAdvisoryDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.References is null || dto.References.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AdvisoryReference>(); | ||||
|         } | ||||
|  | ||||
|         var references = new List<AdvisoryReference>(); | ||||
|         foreach (var reference in dto.References) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(reference.Url)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             try | ||||
|             { | ||||
|                 var provenance = new AdvisoryProvenance( | ||||
|                     DebianConnectorPlugin.SourceName, | ||||
|                     "reference", | ||||
|                     reference.Url, | ||||
|                     recordedAt); | ||||
|  | ||||
|                 references.Add(new AdvisoryReference( | ||||
|                     reference.Url, | ||||
|                     NormalizeReferenceKind(reference.Kind), | ||||
|                     reference.Kind, | ||||
|                     reference.Title, | ||||
|                     provenance)); | ||||
|             } | ||||
|             catch (ArgumentException) | ||||
|             { | ||||
|                 // Ignore malformed URLs while keeping the rest of the advisory intact. | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return references.Count == 0 | ||||
|             ? Array.Empty<AdvisoryReference>() | ||||
|             : references | ||||
|                 .OrderBy(r => r.Url, StringComparer.OrdinalIgnoreCase) | ||||
|                 .ToArray(); | ||||
|     } | ||||
|  | ||||
|     private static string? NormalizeReferenceKind(string? value) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return value.Trim().ToLowerInvariant() switch | ||||
|         { | ||||
|             "advisory" or "dsa" => "advisory", | ||||
|             "cve" => "cve", | ||||
|             "patch" => "patch", | ||||
|             _ => null, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static AdvisoryProvenance BuildPackageProvenance(DebianPackageStateDto package, DateTimeOffset recordedAt) | ||||
|         => new(DebianConnectorPlugin.SourceName, "affected", $"{package.Package}:{package.Release}", recordedAt); | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(DebianAdvisoryDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.Packages is null || dto.Packages.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         var packages = new List<AffectedPackage>(dto.Packages.Count); | ||||
|         foreach (var package in dto.Packages) | ||||
|         { | ||||
|             if (string.IsNullOrWhiteSpace(package.Package)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var provenance = new[] { BuildPackageProvenance(package, recordedAt) }; | ||||
|             var ranges = BuildVersionRanges(package, recordedAt); | ||||
|  | ||||
|             packages.Add(new AffectedPackage( | ||||
|                 AffectedPackageTypes.Deb, | ||||
|                 identifier: package.Package.Trim(), | ||||
|                 platform: package.Release, | ||||
|                 versionRanges: ranges, | ||||
|                 statuses: Array.Empty<AffectedPackageStatus>(), | ||||
|                 provenance: provenance)); | ||||
|         } | ||||
|  | ||||
|         return packages; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(DebianPackageStateDto package, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         var provenance = new AdvisoryProvenance( | ||||
|             DebianConnectorPlugin.SourceName, | ||||
|             "range", | ||||
|             $"{package.Package}:{package.Release}", | ||||
|             recordedAt); | ||||
|  | ||||
|         var introduced = package.IntroducedVersion; | ||||
|         var fixedVersion = package.FixedVersion; | ||||
|         var lastAffected = package.LastAffectedVersion; | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(introduced) && string.IsNullOrWhiteSpace(fixedVersion) && string.IsNullOrWhiteSpace(lastAffected)) | ||||
|         { | ||||
|             return Array.Empty<AffectedVersionRange>(); | ||||
|         } | ||||
|  | ||||
|         var extensions = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|         { | ||||
|             ["debian.release"] = package.Release, | ||||
|             ["debian.status"] = package.Status | ||||
|         }; | ||||
|  | ||||
|         AddExtension(extensions, "debian.introduced", introduced); | ||||
|         AddExtension(extensions, "debian.fixed", fixedVersion); | ||||
|         AddExtension(extensions, "debian.lastAffected", lastAffected); | ||||
|  | ||||
|         var primitives = BuildEvrPrimitives(introduced, fixedVersion, lastAffected); | ||||
|         return new[] | ||||
|         { | ||||
|             new AffectedVersionRange( | ||||
|                 rangeKind: "evr", | ||||
|                 introducedVersion: introduced, | ||||
|                 fixedVersion: fixedVersion, | ||||
|                 lastAffectedVersion: lastAffected, | ||||
|                 rangeExpression: BuildRangeExpression(introduced, fixedVersion, lastAffected), | ||||
|                 provenance: provenance, | ||||
|                 primitives: primitives is null && extensions.Count == 0 | ||||
|                     ? null | ||||
|                     : new RangePrimitives( | ||||
|                         SemVer: null, | ||||
|                         Nevra: null, | ||||
|                         Evr: primitives, | ||||
|                         VendorExtensions: extensions.Count == 0 ? null : extensions)) | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static EvrPrimitive? BuildEvrPrimitives(string? introduced, string? fixedVersion, string? lastAffected) | ||||
|     { | ||||
|         var introducedComponent = ParseEvr(introduced); | ||||
|         var fixedComponent = ParseEvr(fixedVersion); | ||||
|         var lastAffectedComponent = ParseEvr(lastAffected); | ||||
|  | ||||
|         if (introducedComponent is null && fixedComponent is null && lastAffectedComponent is null) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return new EvrPrimitive(introducedComponent, fixedComponent, lastAffectedComponent); | ||||
|     } | ||||
|  | ||||
|     private static EvrComponent? ParseEvr(string? value) | ||||
|     { | ||||
|         if (!DebianEvr.TryParse(value, out var evr) || evr is null) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return new EvrComponent( | ||||
|             evr.Epoch, | ||||
|             evr.Version, | ||||
|             evr.Revision.Length == 0 ? null : evr.Revision); | ||||
|     } | ||||
|  | ||||
|     private static string? BuildRangeExpression(string? introduced, string? fixedVersion, string? lastAffected) | ||||
|     { | ||||
|         var parts = new List<string>(); | ||||
|         if (!string.IsNullOrWhiteSpace(introduced)) | ||||
|         { | ||||
|             parts.Add($"introduced:{introduced.Trim()}"); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(fixedVersion)) | ||||
|         { | ||||
|             parts.Add($"fixed:{fixedVersion.Trim()}"); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(lastAffected)) | ||||
|         { | ||||
|             parts.Add($"last:{lastAffected.Trim()}"); | ||||
|         } | ||||
|  | ||||
|         return parts.Count == 0 ? null : string.Join(" ", parts); | ||||
|     } | ||||
|  | ||||
|     private static void AddExtension(IDictionary<string, string> extensions, string key, string? value) | ||||
|     { | ||||
|         if (!string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             extensions[key] = value.Trim(); | ||||
|         } | ||||
|     } | ||||
| } | ||||
							
								
								
									
										46
									
								
								src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										46
									
								
								src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,46 @@ | ||||
| using System; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Feedser.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Feedser.Source.Distro.Debian; | ||||
|  | ||||
| internal static class DebianJobKinds | ||||
| { | ||||
|     public const string Fetch = "source:debian:fetch"; | ||||
|     public const string Parse = "source:debian:parse"; | ||||
|     public const string Map = "source:debian:map"; | ||||
| } | ||||
|  | ||||
| internal sealed class DebianFetchJob : IJob | ||||
| { | ||||
|     private readonly DebianConnector _connector; | ||||
|  | ||||
|     public DebianFetchJob(DebianConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.FetchAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class DebianParseJob : IJob | ||||
| { | ||||
|     private readonly DebianConnector _connector; | ||||
|  | ||||
|     public DebianParseJob(DebianConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.ParseAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class DebianMapJob : IJob | ||||
| { | ||||
|     private readonly DebianConnector _connector; | ||||
|  | ||||
|     public DebianMapJob(DebianConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.MapAsync(context.Services, cancellationToken); | ||||
| } | ||||
Some files were not shown because too many files have changed in this diff Show More
		Reference in New Issue
	
	Block a user