Rename Concelier Source modules to Connector
This commit is contained in:
		
							
								
								
									
										39
									
								
								src/StellaOps.Concelier.Connector.Ghsa/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										39
									
								
								src/StellaOps.Concelier.Connector.Ghsa/AGENTS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,39 @@ | ||||
| # AGENTS | ||||
| ## Role | ||||
| Implement a connector for GitHub Security Advisories (GHSA) when we need to ingest GHSA content directly (instead of crosswalking via OSV/NVD). | ||||
|  | ||||
| ## Scope | ||||
| - Determine the optimal GHSA data source (GraphQL API, REST, or ecosystem export) and required authentication. | ||||
| - Implement fetch logic with pagination, updated-since filtering, and cursor persistence. | ||||
| - Parse GHSA records (identifiers, summaries, affected packages, versions, references, severity). | ||||
| - Map advisories into canonical `Advisory` objects with aliases, references, affected packages, and range primitives. | ||||
| - Provide deterministic fixtures and regression tests for the full pipeline. | ||||
|  | ||||
| ## Participants | ||||
| - `Source.Common` (HTTP clients, fetch service, DTO storage). | ||||
| - `Storage.Mongo` (raw/document/DTO/advisory stores and source state). | ||||
| - `Concelier.Models` (canonical advisory types). | ||||
| - `Concelier.Testing` (integration harness, snapshot helpers). | ||||
|  | ||||
| ## Interfaces & Contracts | ||||
| - Job kinds: `ghsa:fetch`, `ghsa:parse`, `ghsa:map`. | ||||
| - Support GitHub API authentication & rate limiting (token, retry/backoff). | ||||
| - Alias set must include GHSA IDs and linked CVE IDs. | ||||
|  | ||||
| ## In/Out of scope | ||||
| In scope: | ||||
| - Full GHSA connector implementation with range primitives and provenance instrumentation. | ||||
|  | ||||
| Out of scope: | ||||
| - Repo-specific advisory ingest (handled via GitHub repo exports). | ||||
| - Downstream ecosystem-specific enrichments. | ||||
|  | ||||
| ## Observability & Security Expectations | ||||
| - Log fetch pagination, throttling, and mapping stats. | ||||
| - Handle GitHub API rate limits with exponential backoff and `Retry-After`. | ||||
| - Sanitize/validate payloads before persistence. | ||||
|  | ||||
| ## Tests | ||||
| - Add `StellaOps.Concelier.Connector.Ghsa.Tests` with canned GraphQL/REST fixtures. | ||||
| - Snapshot canonical advisories; enable fixture regeneration with env flag. | ||||
| - Confirm deterministic ordering/time normalisation. | ||||
| @@ -0,0 +1,75 @@ | ||||
| using System.Diagnostics.CodeAnalysis; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Configuration; | ||||
|  | ||||
| public sealed class GhsaOptions | ||||
| { | ||||
|     public static string HttpClientName => "source.ghsa"; | ||||
|  | ||||
|     public Uri BaseEndpoint { get; set; } = new("https://api.github.com/", UriKind.Absolute); | ||||
|  | ||||
|     public string ApiToken { get; set; } = string.Empty; | ||||
|  | ||||
|     public int PageSize { get; set; } = 50; | ||||
|  | ||||
|     public int MaxPagesPerFetch { get; set; } = 5; | ||||
|  | ||||
|     public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); | ||||
|  | ||||
|     public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(200); | ||||
|  | ||||
|     public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); | ||||
|  | ||||
|     public int RateLimitWarningThreshold { get; set; } = 500; | ||||
|  | ||||
|     public TimeSpan SecondaryRateLimitBackoff { get; set; } = TimeSpan.FromMinutes(2); | ||||
|  | ||||
|     [MemberNotNull(nameof(BaseEndpoint), nameof(ApiToken))] | ||||
|     public void Validate() | ||||
|     { | ||||
|         if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) | ||||
|         { | ||||
|             throw new InvalidOperationException("BaseEndpoint must be an absolute URI."); | ||||
|         } | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(ApiToken)) | ||||
|         { | ||||
|             throw new InvalidOperationException("ApiToken must be provided."); | ||||
|         } | ||||
|  | ||||
|         if (PageSize is < 1 or > 100) | ||||
|         { | ||||
|             throw new InvalidOperationException("PageSize must be between 1 and 100."); | ||||
|         } | ||||
|  | ||||
|         if (MaxPagesPerFetch <= 0) | ||||
|         { | ||||
|             throw new InvalidOperationException("MaxPagesPerFetch must be positive."); | ||||
|         } | ||||
|  | ||||
|         if (InitialBackfill < TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("InitialBackfill cannot be negative."); | ||||
|         } | ||||
|  | ||||
|         if (RequestDelay < TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("RequestDelay cannot be negative."); | ||||
|         } | ||||
|  | ||||
|         if (FailureBackoff <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("FailureBackoff must be greater than zero."); | ||||
|         } | ||||
|  | ||||
|         if (RateLimitWarningThreshold < 0) | ||||
|         { | ||||
|             throw new InvalidOperationException("RateLimitWarningThreshold cannot be negative."); | ||||
|         } | ||||
|  | ||||
|         if (SecondaryRateLimitBackoff <= TimeSpan.Zero) | ||||
|         { | ||||
|             throw new InvalidOperationException("SecondaryRateLimitBackoff must be greater than zero."); | ||||
|         } | ||||
|     } | ||||
| } | ||||
							
								
								
									
										547
									
								
								src/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										547
									
								
								src/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,547 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Linq; | ||||
| using System.Net.Http; | ||||
| using System.Text.Json; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using MongoDB.Bson; | ||||
| using StellaOps.Concelier.Models; | ||||
| using StellaOps.Concelier.Connector.Common; | ||||
| using StellaOps.Concelier.Connector.Common.Fetch; | ||||
| using StellaOps.Concelier.Connector.Ghsa.Configuration; | ||||
| using StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
| using StellaOps.Concelier.Storage.Mongo; | ||||
| using StellaOps.Concelier.Storage.Mongo.Advisories; | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
| using StellaOps.Concelier.Storage.Mongo.Dtos; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa; | ||||
|  | ||||
| public sealed class GhsaConnector : IFeedConnector | ||||
| { | ||||
|     private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) | ||||
|     { | ||||
|         PropertyNameCaseInsensitive = true, | ||||
|         WriteIndented = false, | ||||
|     }; | ||||
|  | ||||
|     private readonly SourceFetchService _fetchService; | ||||
|     private readonly RawDocumentStorage _rawDocumentStorage; | ||||
|     private readonly IDocumentStore _documentStore; | ||||
|     private readonly IDtoStore _dtoStore; | ||||
|     private readonly IAdvisoryStore _advisoryStore; | ||||
|     private readonly ISourceStateRepository _stateRepository; | ||||
|     private readonly GhsaOptions _options; | ||||
|     private readonly GhsaDiagnostics _diagnostics; | ||||
|     private readonly TimeProvider _timeProvider; | ||||
|     private readonly ILogger<GhsaConnector> _logger; | ||||
|     private readonly object _rateLimitWarningLock = new(); | ||||
|     private readonly Dictionary<(string Phase, string Resource), bool> _rateLimitWarnings = new(); | ||||
|  | ||||
|     public GhsaConnector( | ||||
|         SourceFetchService fetchService, | ||||
|         RawDocumentStorage rawDocumentStorage, | ||||
|         IDocumentStore documentStore, | ||||
|         IDtoStore dtoStore, | ||||
|         IAdvisoryStore advisoryStore, | ||||
|         ISourceStateRepository stateRepository, | ||||
|         IOptions<GhsaOptions> options, | ||||
|         GhsaDiagnostics diagnostics, | ||||
|         TimeProvider? timeProvider, | ||||
|         ILogger<GhsaConnector> logger) | ||||
|     { | ||||
|         _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); | ||||
|         _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); | ||||
|         _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); | ||||
|         _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); | ||||
|         _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); | ||||
|         _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); | ||||
|         _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); | ||||
|         _options.Validate(); | ||||
|         _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); | ||||
|         _timeProvider = timeProvider ?? TimeProvider.System; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public string SourceName => GhsaConnectorPlugin.SourceName; | ||||
|  | ||||
|     public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var now = _timeProvider.GetUtcNow(); | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         var pendingDocuments = cursor.PendingDocuments.ToHashSet(); | ||||
|         var pendingMappings = cursor.PendingMappings.ToHashSet(); | ||||
|  | ||||
|         var since = cursor.CurrentWindowStart ?? cursor.LastUpdatedExclusive ?? now - _options.InitialBackfill; | ||||
|         if (since > now) | ||||
|         { | ||||
|             since = now; | ||||
|         } | ||||
|  | ||||
|         var until = cursor.CurrentWindowEnd ?? now; | ||||
|         if (until <= since) | ||||
|         { | ||||
|             until = since + TimeSpan.FromMinutes(1); | ||||
|         } | ||||
|  | ||||
|         var page = cursor.NextPage <= 0 ? 1 : cursor.NextPage; | ||||
|         var pagesFetched = 0; | ||||
|         var hasMore = true; | ||||
|         var rateLimitHit = false; | ||||
|         DateTimeOffset? maxUpdated = cursor.LastUpdatedExclusive; | ||||
|  | ||||
|         while (hasMore && pagesFetched < _options.MaxPagesPerFetch) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var listUri = BuildListUri(since, until, page, _options.PageSize); | ||||
|             var metadata = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|             { | ||||
|                 ["since"] = since.ToString("O"), | ||||
|                 ["until"] = until.ToString("O"), | ||||
|                 ["page"] = page.ToString(CultureInfo.InvariantCulture), | ||||
|                 ["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), | ||||
|             }; | ||||
|  | ||||
|             SourceFetchContentResult listResult; | ||||
|             try | ||||
|             { | ||||
|                 _diagnostics.FetchAttempt(); | ||||
|                 listResult = await _fetchService.FetchContentAsync( | ||||
|                     new SourceFetchRequest( | ||||
|                         GhsaOptions.HttpClientName, | ||||
|                         SourceName, | ||||
|                         listUri) | ||||
|                     { | ||||
|                         Metadata = metadata, | ||||
|                         AcceptHeaders = new[] { "application/vnd.github+json" }, | ||||
|                     }, | ||||
|                     cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (HttpRequestException ex) | ||||
|             { | ||||
|                 _diagnostics.FetchFailure(); | ||||
|                 await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             if (listResult.IsNotModified) | ||||
|             { | ||||
|                 _diagnostics.FetchUnchanged(); | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             if (!listResult.IsSuccess || listResult.Content is null) | ||||
|             { | ||||
|                 _diagnostics.FetchFailure(); | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             var deferList = await ApplyRateLimitAsync(listResult.Headers, "list", cancellationToken).ConfigureAwait(false); | ||||
|             if (deferList) | ||||
|             { | ||||
|                 rateLimitHit = true; | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             var pageModel = GhsaListParser.Parse(listResult.Content, page, _options.PageSize); | ||||
|  | ||||
|             if (pageModel.Items.Count == 0) | ||||
|             { | ||||
|                 hasMore = false; | ||||
|             } | ||||
|  | ||||
|             foreach (var item in pageModel.Items) | ||||
|             { | ||||
|                 cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|                 var detailUri = BuildDetailUri(item.GhsaId); | ||||
|                 var detailMetadata = new Dictionary<string, string>(StringComparer.Ordinal) | ||||
|                 { | ||||
|                     ["ghsaId"] = item.GhsaId, | ||||
|                     ["page"] = page.ToString(CultureInfo.InvariantCulture), | ||||
|                     ["since"] = since.ToString("O"), | ||||
|                     ["until"] = until.ToString("O"), | ||||
|                 }; | ||||
|  | ||||
|                 SourceFetchResult detailResult; | ||||
|                 try | ||||
|                 { | ||||
|                     detailResult = await _fetchService.FetchAsync( | ||||
|                         new SourceFetchRequest( | ||||
|                             GhsaOptions.HttpClientName, | ||||
|                             SourceName, | ||||
|                             detailUri) | ||||
|                         { | ||||
|                             Metadata = detailMetadata, | ||||
|                             AcceptHeaders = new[] { "application/vnd.github+json" }, | ||||
|                         }, | ||||
|                         cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|                 catch (HttpRequestException ex) | ||||
|                 { | ||||
|                     _diagnostics.FetchFailure(); | ||||
|                     _logger.LogWarning(ex, "Failed fetching GHSA advisory {GhsaId}", item.GhsaId); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (detailResult.IsNotModified) | ||||
|                 { | ||||
|                     _diagnostics.FetchUnchanged(); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 if (!detailResult.IsSuccess || detailResult.Document is null) | ||||
|                 { | ||||
|                     _diagnostics.FetchFailure(); | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 _diagnostics.FetchDocument(); | ||||
|                 pendingDocuments.Add(detailResult.Document.Id); | ||||
|                 pendingMappings.Add(detailResult.Document.Id); | ||||
|  | ||||
|                 var deferDetail = await ApplyRateLimitAsync(detailResult.Document.Headers, "detail", cancellationToken).ConfigureAwait(false); | ||||
|                 if (deferDetail) | ||||
|                 { | ||||
|                     rateLimitHit = true; | ||||
|                     break; | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             if (rateLimitHit) | ||||
|             { | ||||
|                 break; | ||||
|             } | ||||
|  | ||||
|             if (pageModel.MaxUpdated.HasValue) | ||||
|             { | ||||
|                 if (!maxUpdated.HasValue || pageModel.MaxUpdated > maxUpdated) | ||||
|                 { | ||||
|                     maxUpdated = pageModel.MaxUpdated; | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             hasMore = pageModel.HasMorePages; | ||||
|             page = pageModel.NextPageCandidate; | ||||
|             pagesFetched++; | ||||
|  | ||||
|             if (!rateLimitHit && hasMore && _options.RequestDelay > TimeSpan.Zero) | ||||
|             { | ||||
|                 await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor | ||||
|             .WithPendingDocuments(pendingDocuments) | ||||
|             .WithPendingMappings(pendingMappings); | ||||
|  | ||||
|         if (hasMore || rateLimitHit) | ||||
|         { | ||||
|             updatedCursor = updatedCursor | ||||
|                 .WithCurrentWindowStart(since) | ||||
|                 .WithCurrentWindowEnd(until) | ||||
|                 .WithNextPage(page); | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             var nextSince = maxUpdated ?? until; | ||||
|             updatedCursor = updatedCursor | ||||
|                 .WithLastUpdatedExclusive(nextSince) | ||||
|                 .WithCurrentWindowStart(null) | ||||
|                 .WithCurrentWindowEnd(null) | ||||
|                 .WithNextPage(1); | ||||
|         } | ||||
|  | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingDocuments.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var remainingDocuments = cursor.PendingDocuments.ToList(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingDocuments) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             if (document is null) | ||||
|             { | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (!document.GridFsId.HasValue) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogWarning("GHSA document {DocumentId} missing GridFS content", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             byte[] rawBytes; | ||||
|             try | ||||
|             { | ||||
|                 rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _diagnostics.ParseFailure(); | ||||
|                 _logger.LogError(ex, "Unable to download GHSA raw document {DocumentId}", documentId); | ||||
|                 throw; | ||||
|             } | ||||
|  | ||||
|             GhsaRecordDto dto; | ||||
|             try | ||||
|             { | ||||
|                 dto = GhsaRecordParser.Parse(rawBytes); | ||||
|             } | ||||
|             catch (JsonException ex) | ||||
|             { | ||||
|                 _diagnostics.ParseQuarantine(); | ||||
|                 _logger.LogError(ex, "Malformed GHSA JSON for {DocumentId}", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 remainingDocuments.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); | ||||
|             var dtoRecord = new DtoRecord( | ||||
|                 Guid.NewGuid(), | ||||
|                 document.Id, | ||||
|                 SourceName, | ||||
|                 "ghsa/1.0", | ||||
|                 payload, | ||||
|                 _timeProvider.GetUtcNow()); | ||||
|  | ||||
|             await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             remainingDocuments.Remove(documentId); | ||||
|             _diagnostics.ParseSuccess(); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingDocuments(remainingDocuments); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|  | ||||
|         var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); | ||||
|         if (cursor.PendingMappings.Count == 0) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         var pendingMappings = cursor.PendingMappings.ToList(); | ||||
|  | ||||
|         foreach (var documentId in cursor.PendingMappings) | ||||
|         { | ||||
|             cancellationToken.ThrowIfCancellationRequested(); | ||||
|  | ||||
|             var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|             var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|             if (dtoRecord is null || document is null) | ||||
|             { | ||||
|                 _logger.LogWarning("Skipping GHSA mapping for {DocumentId}: DTO or document missing", documentId); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             GhsaRecordDto dto; | ||||
|             try | ||||
|             { | ||||
|                 dto = JsonSerializer.Deserialize<GhsaRecordDto>(dtoRecord.Payload.ToJson(), SerializerOptions) | ||||
|                     ?? throw new InvalidOperationException("Deserialized DTO was null."); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogError(ex, "Failed to deserialize GHSA DTO for {DocumentId}", documentId); | ||||
|                 await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); | ||||
|                 pendingMappings.Remove(documentId); | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var advisory = GhsaMapper.Map(dto, document, dtoRecord.ValidatedAt); | ||||
|  | ||||
|             if (advisory.CvssMetrics.IsEmpty && !string.IsNullOrWhiteSpace(advisory.CanonicalMetricId)) | ||||
|             { | ||||
|                 var fallbackSeverity = string.IsNullOrWhiteSpace(advisory.Severity) | ||||
|                     ? "unknown" | ||||
|                     : advisory.Severity!; | ||||
|                 _diagnostics.CanonicalMetricFallback(advisory.CanonicalMetricId!, fallbackSeverity); | ||||
|                 if (_logger.IsEnabled(LogLevel.Debug)) | ||||
|                 { | ||||
|                     _logger.LogDebug( | ||||
|                         "GHSA {GhsaId} emitted canonical metric fallback {CanonicalMetricId} (severity {Severity})", | ||||
|                         advisory.AdvisoryKey, | ||||
|                         advisory.CanonicalMetricId, | ||||
|                         fallbackSeverity); | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); | ||||
|             await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); | ||||
|             pendingMappings.Remove(documentId); | ||||
|             _diagnostics.MapSuccess(1); | ||||
|         } | ||||
|  | ||||
|         var updatedCursor = cursor.WithPendingMappings(pendingMappings); | ||||
|         await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private static Uri BuildListUri(DateTimeOffset since, DateTimeOffset until, int page, int pageSize) | ||||
|     { | ||||
|         var query = $"updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(until.ToString("O"))}&page={page}&per_page={pageSize}"; | ||||
|         return new Uri($"security/advisories?{query}", UriKind.Relative); | ||||
|     } | ||||
|  | ||||
|     private static Uri BuildDetailUri(string ghsaId) | ||||
|     { | ||||
|         var encoded = Uri.EscapeDataString(ghsaId); | ||||
|         return new Uri($"security/advisories/{encoded}", UriKind.Relative); | ||||
|     } | ||||
|  | ||||
|     private async Task<GhsaCursor> GetCursorAsync(CancellationToken cancellationToken) | ||||
|     { | ||||
|         var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); | ||||
|         return state is null ? GhsaCursor.Empty : GhsaCursor.FromBson(state.Cursor); | ||||
|     } | ||||
|  | ||||
|     private async Task UpdateCursorAsync(GhsaCursor cursor, CancellationToken cancellationToken) | ||||
|     { | ||||
|         await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private bool ShouldLogRateLimitWarning(in GhsaRateLimitSnapshot snapshot, out bool recovered) | ||||
|     { | ||||
|         recovered = false; | ||||
|  | ||||
|         if (!snapshot.Remaining.HasValue) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         var key = (snapshot.Phase, snapshot.Resource ?? "global"); | ||||
|         var warn = snapshot.Remaining.Value <= _options.RateLimitWarningThreshold; | ||||
|  | ||||
|         lock (_rateLimitWarningLock) | ||||
|         { | ||||
|             var previouslyWarned = _rateLimitWarnings.TryGetValue(key, out var flagged) && flagged; | ||||
|  | ||||
|             if (warn) | ||||
|             { | ||||
|                 if (previouslyWarned) | ||||
|                 { | ||||
|                     return false; | ||||
|                 } | ||||
|  | ||||
|                 _rateLimitWarnings[key] = true; | ||||
|                 return true; | ||||
|             } | ||||
|  | ||||
|             if (previouslyWarned) | ||||
|             { | ||||
|                 _rateLimitWarnings.Remove(key); | ||||
|                 recovered = true; | ||||
|             } | ||||
|  | ||||
|             return false; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static double? CalculateHeadroomPercentage(in GhsaRateLimitSnapshot snapshot) | ||||
|     { | ||||
|         if (!snapshot.Limit.HasValue || !snapshot.Remaining.HasValue) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var limit = snapshot.Limit.Value; | ||||
|         if (limit <= 0) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return (double)snapshot.Remaining.Value / limit * 100d; | ||||
|     } | ||||
|  | ||||
|     private static string FormatHeadroom(double? headroomPct) | ||||
|         => headroomPct.HasValue ? $" (headroom {headroomPct.Value:F1}%)" : string.Empty; | ||||
|  | ||||
|     private async Task<bool> ApplyRateLimitAsync(IReadOnlyDictionary<string, string>? headers, string phase, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var snapshot = GhsaRateLimitParser.TryParse(headers, _timeProvider.GetUtcNow(), phase); | ||||
|         if (snapshot is null || !snapshot.Value.HasData) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         _diagnostics.RecordRateLimit(snapshot.Value); | ||||
|  | ||||
|         var headroomPct = CalculateHeadroomPercentage(snapshot.Value); | ||||
|         if (ShouldLogRateLimitWarning(snapshot.Value, out var recovered)) | ||||
|         { | ||||
|             var resetMessage = snapshot.Value.ResetAfter.HasValue | ||||
|                 ? $" (resets in {snapshot.Value.ResetAfter.Value:c})" | ||||
|                 : snapshot.Value.ResetAt.HasValue ? $" (resets at {snapshot.Value.ResetAt.Value:O})" : string.Empty; | ||||
|  | ||||
|             _logger.LogWarning( | ||||
|                 "GHSA rate limit warning: remaining {Remaining} of {Limit} for {Phase} {Resource}{ResetMessage}{Headroom}", | ||||
|                 snapshot.Value.Remaining, | ||||
|                 snapshot.Value.Limit, | ||||
|                 phase, | ||||
|                 snapshot.Value.Resource ?? "global", | ||||
|                 resetMessage, | ||||
|                 FormatHeadroom(headroomPct)); | ||||
|         } | ||||
|         else if (recovered) | ||||
|         { | ||||
|             _logger.LogInformation( | ||||
|                 "GHSA rate limit recovered for {Phase} {Resource}: remaining {Remaining} of {Limit}{Headroom}", | ||||
|                 phase, | ||||
|                 snapshot.Value.Resource ?? "global", | ||||
|                 snapshot.Value.Remaining, | ||||
|                 snapshot.Value.Limit, | ||||
|                 FormatHeadroom(headroomPct)); | ||||
|         } | ||||
|  | ||||
|         if (snapshot.Value.Remaining.HasValue && snapshot.Value.Remaining.Value <= 0) | ||||
|         { | ||||
|             _diagnostics.RateLimitExhausted(phase); | ||||
|             var delay = snapshot.Value.RetryAfter ?? snapshot.Value.ResetAfter ?? _options.SecondaryRateLimitBackoff; | ||||
|  | ||||
|             if (delay > TimeSpan.Zero) | ||||
|             { | ||||
|                 _logger.LogWarning( | ||||
|                     "GHSA rate limit exhausted for {Phase} {Resource}; delaying {Delay}{Headroom}", | ||||
|                     phase, | ||||
|                     snapshot.Value.Resource ?? "global", | ||||
|                     delay, | ||||
|                     FormatHeadroom(headroomPct)); | ||||
|                 await Task.Delay(delay, cancellationToken).ConfigureAwait(false); | ||||
|             } | ||||
|  | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         return false; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,19 @@ | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.Plugin; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa; | ||||
|  | ||||
| public sealed class GhsaConnectorPlugin : IConnectorPlugin | ||||
| { | ||||
|     public const string SourceName = "ghsa"; | ||||
|  | ||||
|     public string Name => SourceName; | ||||
|  | ||||
|     public bool IsAvailable(IServiceProvider services) => services is not null; | ||||
|  | ||||
|     public IFeedConnector Create(IServiceProvider services) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         return ActivatorUtilities.CreateInstance<GhsaConnector>(services); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,53 @@ | ||||
| using System; | ||||
| using Microsoft.Extensions.Configuration; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using StellaOps.DependencyInjection; | ||||
| using StellaOps.Concelier.Core.Jobs; | ||||
| using StellaOps.Concelier.Connector.Ghsa.Configuration; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa; | ||||
|  | ||||
| public sealed class GhsaDependencyInjectionRoutine : IDependencyInjectionRoutine | ||||
| { | ||||
|     private const string ConfigurationSection = "concelier:sources:ghsa"; | ||||
|     private const string FetchCron = "1,11,21,31,41,51 * * * *"; | ||||
|     private const string ParseCron = "3,13,23,33,43,53 * * * *"; | ||||
|     private const string MapCron = "5,15,25,35,45,55 * * * *"; | ||||
|  | ||||
|     private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6); | ||||
|     private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(5); | ||||
|     private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(5); | ||||
|     private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(4); | ||||
|  | ||||
|     public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configuration); | ||||
|  | ||||
|         services.AddGhsaConnector(options => | ||||
|         { | ||||
|             configuration.GetSection(ConfigurationSection).Bind(options); | ||||
|             options.Validate(); | ||||
|         }); | ||||
|  | ||||
|         var scheduler = new JobSchedulerBuilder(services); | ||||
|         scheduler | ||||
|             .AddJob<GhsaFetchJob>( | ||||
|                 GhsaJobKinds.Fetch, | ||||
|                 cronExpression: FetchCron, | ||||
|                 timeout: FetchTimeout, | ||||
|                 leaseDuration: LeaseDuration) | ||||
|             .AddJob<GhsaParseJob>( | ||||
|                 GhsaJobKinds.Parse, | ||||
|                 cronExpression: ParseCron, | ||||
|                 timeout: ParseTimeout, | ||||
|                 leaseDuration: LeaseDuration) | ||||
|             .AddJob<GhsaMapJob>( | ||||
|                 GhsaJobKinds.Map, | ||||
|                 cronExpression: MapCron, | ||||
|                 timeout: MapTimeout, | ||||
|                 leaseDuration: LeaseDuration); | ||||
|  | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,37 @@ | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Concelier.Connector.Common.Http; | ||||
| using StellaOps.Concelier.Connector.Ghsa.Configuration; | ||||
| using StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa; | ||||
|  | ||||
| public static class GhsaServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddGhsaConnector(this IServiceCollection services, Action<GhsaOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(services); | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|  | ||||
|         services.AddOptions<GhsaOptions>() | ||||
|             .Configure(configure) | ||||
|             .PostConfigure(static opts => opts.Validate()); | ||||
|  | ||||
|         services.AddSourceHttpClient(GhsaOptions.HttpClientName, (sp, clientOptions) => | ||||
|         { | ||||
|             var options = sp.GetRequiredService<IOptions<GhsaOptions>>().Value; | ||||
|             clientOptions.BaseAddress = options.BaseEndpoint; | ||||
|             clientOptions.Timeout = TimeSpan.FromSeconds(30); | ||||
|             clientOptions.UserAgent = "StellaOps.Concelier.Ghsa/1.0"; | ||||
|             clientOptions.AllowedHosts.Clear(); | ||||
|             clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); | ||||
|             clientOptions.DefaultRequestHeaders["Accept"] = "application/vnd.github+json"; | ||||
|             clientOptions.DefaultRequestHeaders["Authorization"] = $"Bearer {options.ApiToken}"; | ||||
|             clientOptions.DefaultRequestHeaders["X-GitHub-Api-Version"] = "2022-11-28"; | ||||
|         }); | ||||
|  | ||||
|         services.AddSingleton<GhsaDiagnostics>(); | ||||
|         services.AddTransient<GhsaConnector>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										135
									
								
								src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaCursor.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										135
									
								
								src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaCursor.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,135 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Linq; | ||||
| using MongoDB.Bson; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| internal sealed record GhsaCursor( | ||||
|     DateTimeOffset? LastUpdatedExclusive, | ||||
|     DateTimeOffset? CurrentWindowStart, | ||||
|     DateTimeOffset? CurrentWindowEnd, | ||||
|     int NextPage, | ||||
|     IReadOnlyCollection<Guid> PendingDocuments, | ||||
|     IReadOnlyCollection<Guid> PendingMappings) | ||||
| { | ||||
|     private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); | ||||
|  | ||||
|     public static GhsaCursor Empty { get; } = new( | ||||
|         null, | ||||
|         null, | ||||
|         null, | ||||
|         1, | ||||
|         EmptyGuidList, | ||||
|         EmptyGuidList); | ||||
|  | ||||
|     public BsonDocument ToBsonDocument() | ||||
|     { | ||||
|         var document = new BsonDocument | ||||
|         { | ||||
|             ["nextPage"] = NextPage, | ||||
|             ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), | ||||
|             ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), | ||||
|         }; | ||||
|  | ||||
|         if (LastUpdatedExclusive.HasValue) | ||||
|         { | ||||
|             document["lastUpdatedExclusive"] = LastUpdatedExclusive.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         if (CurrentWindowStart.HasValue) | ||||
|         { | ||||
|             document["currentWindowStart"] = CurrentWindowStart.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         if (CurrentWindowEnd.HasValue) | ||||
|         { | ||||
|             document["currentWindowEnd"] = CurrentWindowEnd.Value.UtcDateTime; | ||||
|         } | ||||
|  | ||||
|         return document; | ||||
|     } | ||||
|  | ||||
|     public static GhsaCursor FromBson(BsonDocument? document) | ||||
|     { | ||||
|         if (document is null || document.ElementCount == 0) | ||||
|         { | ||||
|             return Empty; | ||||
|         } | ||||
|  | ||||
|         var lastUpdatedExclusive = document.TryGetValue("lastUpdatedExclusive", out var lastUpdated) | ||||
|             ? ParseDate(lastUpdated) | ||||
|             : null; | ||||
|         var windowStart = document.TryGetValue("currentWindowStart", out var windowStartValue) | ||||
|             ? ParseDate(windowStartValue) | ||||
|             : null; | ||||
|         var windowEnd = document.TryGetValue("currentWindowEnd", out var windowEndValue) | ||||
|             ? ParseDate(windowEndValue) | ||||
|             : null; | ||||
|         var nextPage = document.TryGetValue("nextPage", out var nextPageValue) && nextPageValue.IsInt32 | ||||
|             ? Math.Max(1, nextPageValue.AsInt32) | ||||
|             : 1; | ||||
|  | ||||
|         var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); | ||||
|         var pendingMappings = ReadGuidArray(document, "pendingMappings"); | ||||
|  | ||||
|         return new GhsaCursor( | ||||
|             lastUpdatedExclusive, | ||||
|             windowStart, | ||||
|             windowEnd, | ||||
|             nextPage, | ||||
|             pendingDocuments, | ||||
|             pendingMappings); | ||||
|     } | ||||
|  | ||||
|     public GhsaCursor WithPendingDocuments(IEnumerable<Guid> ids) | ||||
|         => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; | ||||
|  | ||||
|     public GhsaCursor WithPendingMappings(IEnumerable<Guid> ids) | ||||
|         => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; | ||||
|  | ||||
|     public GhsaCursor WithLastUpdatedExclusive(DateTimeOffset? timestamp) | ||||
|         => this with { LastUpdatedExclusive = timestamp }; | ||||
|  | ||||
|     public GhsaCursor WithCurrentWindowStart(DateTimeOffset? timestamp) | ||||
|         => this with { CurrentWindowStart = timestamp }; | ||||
|  | ||||
|     public GhsaCursor WithCurrentWindowEnd(DateTimeOffset? timestamp) | ||||
|         => this with { CurrentWindowEnd = timestamp }; | ||||
|  | ||||
|     public GhsaCursor WithNextPage(int page) | ||||
|         => this with { NextPage = page < 1 ? 1 : page }; | ||||
|  | ||||
|     private static DateTimeOffset? ParseDate(BsonValue value) | ||||
|     { | ||||
|         return value.BsonType switch | ||||
|         { | ||||
|             BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), | ||||
|             BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), | ||||
|             _ => null, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) | ||||
|     { | ||||
|         if (!document.TryGetValue(field, out var value) || value is not BsonArray array) | ||||
|         { | ||||
|             return EmptyGuidList; | ||||
|         } | ||||
|  | ||||
|         var results = new List<Guid>(array.Count); | ||||
|         foreach (var element in array) | ||||
|         { | ||||
|             if (element is null) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             if (Guid.TryParse(element.ToString(), out var guid)) | ||||
|             { | ||||
|                 results.Add(guid); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return results; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,164 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Diagnostics.Metrics; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| public sealed class GhsaDiagnostics : IDisposable | ||||
| { | ||||
|     private const string MeterName = "StellaOps.Concelier.Connector.Ghsa"; | ||||
|     private const string MeterVersion = "1.0.0"; | ||||
|  | ||||
|     private readonly Meter _meter; | ||||
|     private readonly Counter<long> _fetchAttempts; | ||||
|     private readonly Counter<long> _fetchDocuments; | ||||
|     private readonly Counter<long> _fetchFailures; | ||||
|     private readonly Counter<long> _fetchUnchanged; | ||||
|     private readonly Counter<long> _parseSuccess; | ||||
|     private readonly Counter<long> _parseFailures; | ||||
|     private readonly Counter<long> _parseQuarantine; | ||||
|     private readonly Counter<long> _mapSuccess; | ||||
|     private readonly Histogram<long> _rateLimitRemaining; | ||||
|     private readonly Histogram<long> _rateLimitLimit; | ||||
|     private readonly Histogram<double> _rateLimitResetSeconds; | ||||
|     private readonly Histogram<double> _rateLimitHeadroomPct; | ||||
|     private readonly ObservableGauge<double> _rateLimitHeadroomGauge; | ||||
|     private readonly Counter<long> _rateLimitExhausted; | ||||
|     private readonly Counter<long> _canonicalMetricFallbacks; | ||||
|     private readonly object _rateLimitLock = new(); | ||||
|     private GhsaRateLimitSnapshot? _lastRateLimitSnapshot; | ||||
|     private readonly Dictionary<(string Phase, string? Resource), GhsaRateLimitSnapshot> _rateLimitSnapshots = new(); | ||||
|  | ||||
|     public GhsaDiagnostics() | ||||
|     { | ||||
|         _meter = new Meter(MeterName, MeterVersion); | ||||
|         _fetchAttempts = _meter.CreateCounter<long>("ghsa.fetch.attempts", unit: "operations"); | ||||
|         _fetchDocuments = _meter.CreateCounter<long>("ghsa.fetch.documents", unit: "documents"); | ||||
|         _fetchFailures = _meter.CreateCounter<long>("ghsa.fetch.failures", unit: "operations"); | ||||
|         _fetchUnchanged = _meter.CreateCounter<long>("ghsa.fetch.unchanged", unit: "operations"); | ||||
|         _parseSuccess = _meter.CreateCounter<long>("ghsa.parse.success", unit: "documents"); | ||||
|         _parseFailures = _meter.CreateCounter<long>("ghsa.parse.failures", unit: "documents"); | ||||
|         _parseQuarantine = _meter.CreateCounter<long>("ghsa.parse.quarantine", unit: "documents"); | ||||
|         _mapSuccess = _meter.CreateCounter<long>("ghsa.map.success", unit: "advisories"); | ||||
|         _rateLimitRemaining = _meter.CreateHistogram<long>("ghsa.ratelimit.remaining", unit: "requests"); | ||||
|         _rateLimitLimit = _meter.CreateHistogram<long>("ghsa.ratelimit.limit", unit: "requests"); | ||||
|         _rateLimitResetSeconds = _meter.CreateHistogram<double>("ghsa.ratelimit.reset_seconds", unit: "s"); | ||||
|         _rateLimitHeadroomPct = _meter.CreateHistogram<double>("ghsa.ratelimit.headroom_pct", unit: "percent"); | ||||
|         _rateLimitHeadroomGauge = _meter.CreateObservableGauge("ghsa.ratelimit.headroom_pct_current", ObserveHeadroom, unit: "percent"); | ||||
|         _rateLimitExhausted = _meter.CreateCounter<long>("ghsa.ratelimit.exhausted", unit: "events"); | ||||
|         _canonicalMetricFallbacks = _meter.CreateCounter<long>("ghsa.map.canonical_metric_fallbacks", unit: "advisories"); | ||||
|     } | ||||
|  | ||||
|     public void FetchAttempt() => _fetchAttempts.Add(1); | ||||
|  | ||||
|     public void FetchDocument() => _fetchDocuments.Add(1); | ||||
|  | ||||
|     public void FetchFailure() => _fetchFailures.Add(1); | ||||
|  | ||||
|     public void FetchUnchanged() => _fetchUnchanged.Add(1); | ||||
|  | ||||
|     public void ParseSuccess() => _parseSuccess.Add(1); | ||||
|  | ||||
|     public void ParseFailure() => _parseFailures.Add(1); | ||||
|  | ||||
|     public void ParseQuarantine() => _parseQuarantine.Add(1); | ||||
|  | ||||
|     public void MapSuccess(long count) => _mapSuccess.Add(count); | ||||
|  | ||||
|     internal void RecordRateLimit(GhsaRateLimitSnapshot snapshot) | ||||
|     { | ||||
|         var tags = new KeyValuePair<string, object?>[] | ||||
|         { | ||||
|             new("phase", snapshot.Phase), | ||||
|             new("resource", snapshot.Resource ?? "unknown") | ||||
|         }; | ||||
|  | ||||
|         if (snapshot.Limit.HasValue) | ||||
|         { | ||||
|             _rateLimitLimit.Record(snapshot.Limit.Value, tags); | ||||
|         } | ||||
|  | ||||
|         if (snapshot.Remaining.HasValue) | ||||
|         { | ||||
|             _rateLimitRemaining.Record(snapshot.Remaining.Value, tags); | ||||
|         } | ||||
|  | ||||
|         if (snapshot.ResetAfter.HasValue) | ||||
|         { | ||||
|             _rateLimitResetSeconds.Record(snapshot.ResetAfter.Value.TotalSeconds, tags); | ||||
|         } | ||||
|  | ||||
|         if (TryCalculateHeadroom(snapshot, out var headroom)) | ||||
|         { | ||||
|             _rateLimitHeadroomPct.Record(headroom, tags); | ||||
|         } | ||||
|  | ||||
|         lock (_rateLimitLock) | ||||
|         { | ||||
|             _lastRateLimitSnapshot = snapshot; | ||||
|             _rateLimitSnapshots[(snapshot.Phase, snapshot.Resource)] = snapshot; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     internal void RateLimitExhausted(string phase) | ||||
|         => _rateLimitExhausted.Add(1, new KeyValuePair<string, object?>("phase", phase)); | ||||
|  | ||||
|     public void CanonicalMetricFallback(string canonicalMetricId, string severity) | ||||
|         => _canonicalMetricFallbacks.Add( | ||||
|             1, | ||||
|             new KeyValuePair<string, object?>("canonical_metric_id", canonicalMetricId), | ||||
|             new KeyValuePair<string, object?>("severity", severity), | ||||
|             new KeyValuePair<string, object?>("reason", "no_cvss")); | ||||
|  | ||||
|     internal GhsaRateLimitSnapshot? GetLastRateLimitSnapshot() | ||||
|     { | ||||
|         lock (_rateLimitLock) | ||||
|         { | ||||
|             return _lastRateLimitSnapshot; | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private IEnumerable<Measurement<double>> ObserveHeadroom() | ||||
|     { | ||||
|         lock (_rateLimitLock) | ||||
|         { | ||||
|             if (_rateLimitSnapshots.Count == 0) | ||||
|             { | ||||
|                 yield break; | ||||
|             } | ||||
|  | ||||
|             foreach (var snapshot in _rateLimitSnapshots.Values) | ||||
|             { | ||||
|                 if (TryCalculateHeadroom(snapshot, out var headroom)) | ||||
|                 { | ||||
|                     yield return new Measurement<double>( | ||||
|                         headroom, | ||||
|                         new KeyValuePair<string, object?>("phase", snapshot.Phase), | ||||
|                         new KeyValuePair<string, object?>("resource", snapshot.Resource ?? "unknown")); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private static bool TryCalculateHeadroom(in GhsaRateLimitSnapshot snapshot, out double headroomPct) | ||||
|     { | ||||
|         headroomPct = 0; | ||||
|         if (!snapshot.Limit.HasValue || !snapshot.Remaining.HasValue) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         var limit = snapshot.Limit.Value; | ||||
|         if (limit <= 0) | ||||
|         { | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         headroomPct = (double)snapshot.Remaining.Value / limit * 100d; | ||||
|         return true; | ||||
|     } | ||||
|  | ||||
|     public void Dispose() | ||||
|     { | ||||
|         _meter.Dispose(); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,115 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Text.Json; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| internal static class GhsaListParser | ||||
| { | ||||
|     public static GhsaListPage Parse(ReadOnlySpan<byte> content, int currentPage, int pageSize) | ||||
|     { | ||||
|         using var document = JsonDocument.Parse(content.ToArray()); | ||||
|         var root = document.RootElement; | ||||
|  | ||||
|         var items = new List<GhsaListItem>(); | ||||
|         DateTimeOffset? maxUpdated = null; | ||||
|  | ||||
|         if (root.TryGetProperty("advisories", out var advisories) && advisories.ValueKind == JsonValueKind.Array) | ||||
|         { | ||||
|             foreach (var advisory in advisories.EnumerateArray()) | ||||
|             { | ||||
|                 if (advisory.ValueKind != JsonValueKind.Object) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var id = GetString(advisory, "ghsa_id"); | ||||
|                 if (string.IsNullOrWhiteSpace(id)) | ||||
|                 { | ||||
|                     continue; | ||||
|                 } | ||||
|  | ||||
|                 var updated = GetDate(advisory, "updated_at"); | ||||
|                 if (updated.HasValue && (!maxUpdated.HasValue || updated > maxUpdated)) | ||||
|                 { | ||||
|                     maxUpdated = updated; | ||||
|                 } | ||||
|  | ||||
|                 items.Add(new GhsaListItem(id, updated)); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var hasMorePages = TryDetermineHasMore(root, currentPage, pageSize, items.Count, out var nextPage); | ||||
|  | ||||
|         return new GhsaListPage(items, maxUpdated, hasMorePages, nextPage ?? currentPage + 1); | ||||
|     } | ||||
|  | ||||
|     private static bool TryDetermineHasMore(JsonElement root, int currentPage, int pageSize, int itemCount, out int? nextPage) | ||||
|     { | ||||
|         nextPage = null; | ||||
|  | ||||
|         if (root.TryGetProperty("pagination", out var pagination) && pagination.ValueKind == JsonValueKind.Object) | ||||
|         { | ||||
|             var hasNextPage = pagination.TryGetProperty("has_next_page", out var hasNext) && hasNext.ValueKind == JsonValueKind.True; | ||||
|             if (hasNextPage) | ||||
|             { | ||||
|                 nextPage = currentPage + 1; | ||||
|                 return true; | ||||
|             } | ||||
|  | ||||
|             if (pagination.TryGetProperty("total_pages", out var totalPagesElement) && totalPagesElement.ValueKind == JsonValueKind.Number && totalPagesElement.TryGetInt32(out var totalPages)) | ||||
|             { | ||||
|                 if (currentPage < totalPages) | ||||
|                 { | ||||
|                     nextPage = currentPage + 1; | ||||
|                     return true; | ||||
|                 } | ||||
|             } | ||||
|  | ||||
|             return false; | ||||
|         } | ||||
|  | ||||
|         if (itemCount >= pageSize) | ||||
|         { | ||||
|             nextPage = currentPage + 1; | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         return false; | ||||
|     } | ||||
|  | ||||
|     private static string? GetString(JsonElement element, string propertyName) | ||||
|     { | ||||
|         if (!element.TryGetProperty(propertyName, out var property)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return property.ValueKind switch | ||||
|         { | ||||
|             JsonValueKind.String => property.GetString(), | ||||
|             _ => null, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static DateTimeOffset? GetDate(JsonElement element, string propertyName) | ||||
|     { | ||||
|         var value = GetString(element, propertyName); | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) | ||||
|             ? parsed.ToUniversalTime() | ||||
|             : null; | ||||
|     } | ||||
| } | ||||
|  | ||||
| internal sealed record GhsaListPage( | ||||
|     IReadOnlyList<GhsaListItem> Items, | ||||
|     DateTimeOffset? MaxUpdated, | ||||
|     bool HasMorePages, | ||||
|     int NextPageCandidate); | ||||
|  | ||||
| internal sealed record GhsaListItem(string GhsaId, DateTimeOffset? UpdatedAt); | ||||
							
								
								
									
										447
									
								
								src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaMapper.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										447
									
								
								src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaMapper.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,447 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Collections.Immutable; | ||||
| using System.Linq; | ||||
| using System.Text; | ||||
| using StellaOps.Concelier.Models; | ||||
| using StellaOps.Concelier.Normalization.Cvss; | ||||
| using StellaOps.Concelier.Normalization.SemVer; | ||||
| using StellaOps.Concelier.Storage.Mongo.Documents; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| internal static class GhsaMapper | ||||
| { | ||||
|     private static readonly HashSet<string> SemVerEcosystems = new(StringComparer.OrdinalIgnoreCase) | ||||
|     { | ||||
|         "npm", | ||||
|         "maven", | ||||
|         "pip", | ||||
|         "rubygems", | ||||
|         "composer", | ||||
|         "nuget", | ||||
|         "go", | ||||
|         "cargo", | ||||
|     }; | ||||
|  | ||||
|     public static Advisory Map(GhsaRecordDto dto, DocumentRecord document, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(dto); | ||||
|         ArgumentNullException.ThrowIfNull(document); | ||||
|  | ||||
|         var fetchProvenance = new AdvisoryProvenance( | ||||
|             GhsaConnectorPlugin.SourceName, | ||||
|             "document", | ||||
|             document.Uri, | ||||
|             document.FetchedAt, | ||||
|             new[] { ProvenanceFieldMasks.Advisory }); | ||||
|         var mapProvenance = new AdvisoryProvenance( | ||||
|             GhsaConnectorPlugin.SourceName, | ||||
|             "mapping", | ||||
|             dto.GhsaId, | ||||
|             recordedAt, | ||||
|             new[] { ProvenanceFieldMasks.Advisory }); | ||||
|  | ||||
|         var aliases = dto.Aliases | ||||
|             .Where(static alias => !string.IsNullOrWhiteSpace(alias)) | ||||
|             .Distinct(StringComparer.OrdinalIgnoreCase) | ||||
|             .ToArray(); | ||||
|  | ||||
|         var references = dto.References | ||||
|             .Select(reference => CreateReference(reference, recordedAt)) | ||||
|             .Where(static reference => reference is not null) | ||||
|             .Cast<AdvisoryReference>() | ||||
|             .ToList(); | ||||
|  | ||||
|         var affected = CreateAffectedPackages(dto, recordedAt); | ||||
|         var credits = CreateCredits(dto.Credits, recordedAt); | ||||
|         var weaknesses = CreateWeaknesses(dto.Cwes, recordedAt); | ||||
|         var cvssMetrics = CreateCvssMetrics(dto.Cvss, recordedAt, out var cvssSeverity, out var canonicalMetricId); | ||||
|  | ||||
|         var severityHint = SeverityNormalization.Normalize(dto.Severity); | ||||
|         var cvssSeverityHint = SeverityNormalization.Normalize(dto.Cvss?.Severity); | ||||
|         var severity = severityHint ?? cvssSeverity ?? cvssSeverityHint; | ||||
|  | ||||
|         if (canonicalMetricId is null) | ||||
|         { | ||||
|             var fallbackSeverity = severityHint ?? cvssSeverityHint ?? cvssSeverity; | ||||
|             if (!string.IsNullOrWhiteSpace(fallbackSeverity)) | ||||
|             { | ||||
|                 canonicalMetricId = BuildSeverityCanonicalMetricId(fallbackSeverity); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var summary = dto.Summary ?? dto.Description; | ||||
|         var description = Validation.TrimToNull(dto.Description); | ||||
|  | ||||
|         return new Advisory( | ||||
|             advisoryKey: dto.GhsaId, | ||||
|             title: dto.Summary ?? dto.GhsaId, | ||||
|             summary: summary, | ||||
|             language: "en", | ||||
|             published: dto.PublishedAt, | ||||
|             modified: dto.UpdatedAt ?? dto.PublishedAt, | ||||
|             severity: severity, | ||||
|             exploitKnown: false, | ||||
|             aliases: aliases, | ||||
|             credits: credits, | ||||
|             references: references, | ||||
|             affectedPackages: affected, | ||||
|             cvssMetrics: cvssMetrics, | ||||
|             provenance: new[] { fetchProvenance, mapProvenance }, | ||||
|             description: description, | ||||
|             cwes: weaknesses, | ||||
|             canonicalMetricId: canonicalMetricId); | ||||
|     } | ||||
|  | ||||
|     private static string BuildSeverityCanonicalMetricId(string severity) | ||||
|         => $"{GhsaConnectorPlugin.SourceName}:severity/{severity}"; | ||||
|  | ||||
|     private static AdvisoryReference? CreateReference(GhsaReferenceDto reference, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(reference.Url) || !Validation.LooksLikeHttpUrl(reference.Url)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var kind = reference.Type?.ToLowerInvariant(); | ||||
|  | ||||
|         return new AdvisoryReference( | ||||
|             reference.Url, | ||||
|             kind, | ||||
|             reference.Name, | ||||
|             summary: null, | ||||
|             provenance: new AdvisoryProvenance( | ||||
|                 GhsaConnectorPlugin.SourceName, | ||||
|                 "reference", | ||||
|                 reference.Url, | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.References })); | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AffectedPackage> CreateAffectedPackages(GhsaRecordDto dto, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (dto.Affected.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AffectedPackage>(); | ||||
|         } | ||||
|  | ||||
|         var packages = new List<AffectedPackage>(dto.Affected.Count); | ||||
|         foreach (var affected in dto.Affected) | ||||
|         { | ||||
|             var ecosystem = string.IsNullOrWhiteSpace(affected.Ecosystem) ? "unknown" : affected.Ecosystem.Trim(); | ||||
|             var packageName = string.IsNullOrWhiteSpace(affected.PackageName) ? "unknown-package" : affected.PackageName.Trim(); | ||||
|             var identifier = $"{ecosystem.ToLowerInvariant()}:{packageName}"; | ||||
|  | ||||
|             var provenance = new[] | ||||
|             { | ||||
|                 new AdvisoryProvenance( | ||||
|                     GhsaConnectorPlugin.SourceName, | ||||
|                     "affected", | ||||
|                     identifier, | ||||
|                     recordedAt, | ||||
|                     new[] { ProvenanceFieldMasks.AffectedPackages }), | ||||
|             }; | ||||
|  | ||||
|             var rangeKind = SemVerEcosystems.Contains(ecosystem) ? "semver" : "vendor"; | ||||
|             var packageType = SemVerEcosystems.Contains(ecosystem) ? AffectedPackageTypes.SemVer : AffectedPackageTypes.Vendor; | ||||
|  | ||||
|             var (ranges, normalizedVersions) = SemVerEcosystems.Contains(ecosystem) | ||||
|                 ? CreateSemVerVersionArtifacts(affected, identifier, ecosystem, packageName, recordedAt) | ||||
|                 : CreateVendorVersionArtifacts(affected, rangeKind, identifier, ecosystem, packageName, recordedAt); | ||||
|  | ||||
|             var statuses = new[] | ||||
|             { | ||||
|                 new AffectedPackageStatus( | ||||
|                     "affected", | ||||
|                     new AdvisoryProvenance( | ||||
|                         GhsaConnectorPlugin.SourceName, | ||||
|                         "affected-status", | ||||
|                         identifier, | ||||
|                         recordedAt, | ||||
|                         new[] { ProvenanceFieldMasks.PackageStatuses })), | ||||
|             }; | ||||
|  | ||||
|             packages.Add(new AffectedPackage( | ||||
|                 packageType, | ||||
|                 identifier, | ||||
|                 platform: null, | ||||
|                 versionRanges: ranges, | ||||
|                 statuses: statuses, | ||||
|                 provenance: provenance, | ||||
|                 normalizedVersions: normalizedVersions)); | ||||
|         } | ||||
|  | ||||
|         return packages; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryCredit> CreateCredits(IReadOnlyList<GhsaCreditDto> credits, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (credits.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AdvisoryCredit>(); | ||||
|         } | ||||
|  | ||||
|         var results = new List<AdvisoryCredit>(credits.Count); | ||||
|         foreach (var credit in credits) | ||||
|         { | ||||
|             var displayName = Validation.TrimToNull(credit.Name) ?? Validation.TrimToNull(credit.Login); | ||||
|             if (displayName is null) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var contacts = new List<string>(); | ||||
|             if (!string.IsNullOrWhiteSpace(credit.ProfileUrl) && Validation.LooksLikeHttpUrl(credit.ProfileUrl)) | ||||
|             { | ||||
|                 contacts.Add(credit.ProfileUrl.Trim()); | ||||
|             } | ||||
|             else if (!string.IsNullOrWhiteSpace(credit.Login)) | ||||
|             { | ||||
|                 contacts.Add($"https://github.com/{credit.Login.Trim()}"); | ||||
|             } | ||||
|  | ||||
|             var provenance = new AdvisoryProvenance( | ||||
|                 GhsaConnectorPlugin.SourceName, | ||||
|                 "credit", | ||||
|                 displayName, | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.Credits }); | ||||
|  | ||||
|             results.Add(new AdvisoryCredit(displayName, credit.Type, contacts, provenance)); | ||||
|         } | ||||
|  | ||||
|         return results.Count == 0 ? Array.Empty<AdvisoryCredit>() : results; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<AdvisoryWeakness> CreateWeaknesses(IReadOnlyList<GhsaWeaknessDto> cwes, DateTimeOffset recordedAt) | ||||
|     { | ||||
|         if (cwes.Count == 0) | ||||
|         { | ||||
|             return Array.Empty<AdvisoryWeakness>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<AdvisoryWeakness>(cwes.Count); | ||||
|         foreach (var cwe in cwes) | ||||
|         { | ||||
|             if (cwe is null || string.IsNullOrWhiteSpace(cwe.CweId)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var identifier = cwe.CweId.Trim(); | ||||
|             var provenance = new AdvisoryProvenance( | ||||
|                 GhsaConnectorPlugin.SourceName, | ||||
|                 "weakness", | ||||
|                 identifier, | ||||
|                 recordedAt, | ||||
|                 new[] { ProvenanceFieldMasks.Weaknesses }); | ||||
|  | ||||
|             var provenanceArray = ImmutableArray.Create(provenance); | ||||
|             list.Add(new AdvisoryWeakness( | ||||
|                 taxonomy: "cwe", | ||||
|                 identifier: identifier, | ||||
|                 name: Validation.TrimToNull(cwe.Name), | ||||
|                 uri: BuildCweUrl(identifier), | ||||
|                 provenance: provenanceArray)); | ||||
|         } | ||||
|  | ||||
|         return list.Count == 0 ? Array.Empty<AdvisoryWeakness>() : list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<CvssMetric> CreateCvssMetrics(GhsaCvssDto? cvss, DateTimeOffset recordedAt, out string? severity, out string? canonicalMetricId) | ||||
|     { | ||||
|         severity = null; | ||||
|         canonicalMetricId = null; | ||||
|  | ||||
|         if (cvss is null) | ||||
|         { | ||||
|             return Array.Empty<CvssMetric>(); | ||||
|         } | ||||
|  | ||||
|         var vector = Validation.TrimToNull(cvss.VectorString); | ||||
|         if (!CvssMetricNormalizer.TryNormalize(null, vector, cvss.Score, cvss.Severity, out var normalized)) | ||||
|         { | ||||
|             return Array.Empty<CvssMetric>(); | ||||
|         } | ||||
|  | ||||
|         severity = normalized.BaseSeverity; | ||||
|         canonicalMetricId = $"{normalized.Version}|{normalized.Vector}"; | ||||
|  | ||||
|         var provenance = new AdvisoryProvenance( | ||||
|             GhsaConnectorPlugin.SourceName, | ||||
|             "cvss", | ||||
|             normalized.Vector, | ||||
|             recordedAt, | ||||
|             new[] { ProvenanceFieldMasks.CvssMetrics }); | ||||
|  | ||||
|         return new[] | ||||
|         { | ||||
|             normalized.ToModel(provenance), | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static string? BuildCweUrl(string? cweId) | ||||
|     { | ||||
|         if (string.IsNullOrWhiteSpace(cweId)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var trimmed = cweId.Trim(); | ||||
|         var dashIndex = trimmed.IndexOf('-'); | ||||
|         if (dashIndex < 0 || dashIndex == trimmed.Length - 1) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         var digits = new StringBuilder(); | ||||
|         for (var i = dashIndex + 1; i < trimmed.Length; i++) | ||||
|         { | ||||
|             var ch = trimmed[i]; | ||||
|             if (char.IsDigit(ch)) | ||||
|             { | ||||
|                 digits.Append(ch); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return digits.Length == 0 ? null : $"https://cwe.mitre.org/data/definitions/{digits}.html"; | ||||
|     } | ||||
|  | ||||
|     private static (IReadOnlyList<AffectedVersionRange> Ranges, IReadOnlyList<NormalizedVersionRule> Normalized) CreateSemVerVersionArtifacts( | ||||
|         GhsaAffectedDto affected, | ||||
|         string identifier, | ||||
|         string ecosystem, | ||||
|         string packageName, | ||||
|         DateTimeOffset recordedAt) | ||||
|     { | ||||
|         var note = BuildNormalizedNote(identifier); | ||||
|         var results = SemVerRangeRuleBuilder.Build(affected.VulnerableRange, affected.PatchedVersion, note); | ||||
|  | ||||
|         if (results.Count > 0) | ||||
|         { | ||||
|             var ranges = new List<AffectedVersionRange>(results.Count); | ||||
|             var normalized = new List<NormalizedVersionRule>(results.Count); | ||||
|  | ||||
|             foreach (var result in results) | ||||
|             { | ||||
|                 var primitive = result.Primitive; | ||||
|                 var rangeExpression = ResolveRangeExpression(result.Expression, primitive.ConstraintExpression, affected.VulnerableRange); | ||||
|  | ||||
|                 ranges.Add(new AffectedVersionRange( | ||||
|                     rangeKind: "semver", | ||||
|                     introducedVersion: Validation.TrimToNull(primitive.Introduced), | ||||
|                     fixedVersion: Validation.TrimToNull(primitive.Fixed), | ||||
|                     lastAffectedVersion: Validation.TrimToNull(primitive.LastAffected), | ||||
|                     rangeExpression: rangeExpression, | ||||
|                     provenance: CreateRangeProvenance(identifier, recordedAt), | ||||
|                     primitives: new RangePrimitives( | ||||
|                         SemVer: primitive, | ||||
|                         Nevra: null, | ||||
|                         Evr: null, | ||||
|                         VendorExtensions: CreateVendorExtensions(ecosystem, packageName)))); | ||||
|  | ||||
|                 normalized.Add(result.NormalizedRule); | ||||
|             } | ||||
|  | ||||
|             return (ranges.ToArray(), normalized.ToArray()); | ||||
|         } | ||||
|  | ||||
|         var fallbackRange = CreateFallbackRange("semver", affected, identifier, ecosystem, packageName, recordedAt); | ||||
|         if (fallbackRange is null) | ||||
|         { | ||||
|             return (Array.Empty<AffectedVersionRange>(), Array.Empty<NormalizedVersionRule>()); | ||||
|         } | ||||
|  | ||||
|         var fallbackRule = fallbackRange.ToNormalizedVersionRule(note); | ||||
|         var normalizedFallback = fallbackRule is null | ||||
|             ? Array.Empty<NormalizedVersionRule>() | ||||
|             : new[] { fallbackRule }; | ||||
|  | ||||
|         return (new[] { fallbackRange }, normalizedFallback); | ||||
|     } | ||||
|  | ||||
|     private static (IReadOnlyList<AffectedVersionRange> Ranges, IReadOnlyList<NormalizedVersionRule> Normalized) CreateVendorVersionArtifacts( | ||||
|         GhsaAffectedDto affected, | ||||
|         string rangeKind, | ||||
|         string identifier, | ||||
|         string ecosystem, | ||||
|         string packageName, | ||||
|         DateTimeOffset recordedAt) | ||||
|     { | ||||
|         var range = CreateFallbackRange(rangeKind, affected, identifier, ecosystem, packageName, recordedAt); | ||||
|         if (range is null) | ||||
|         { | ||||
|             return (Array.Empty<AffectedVersionRange>(), Array.Empty<NormalizedVersionRule>()); | ||||
|         } | ||||
|  | ||||
|         return (new[] { range }, Array.Empty<NormalizedVersionRule>()); | ||||
|     } | ||||
|  | ||||
|     private static AffectedVersionRange? CreateFallbackRange( | ||||
|         string rangeKind, | ||||
|         GhsaAffectedDto affected, | ||||
|         string identifier, | ||||
|         string ecosystem, | ||||
|         string packageName, | ||||
|         DateTimeOffset recordedAt) | ||||
|     { | ||||
|         var fixedVersion = Validation.TrimToNull(affected.PatchedVersion); | ||||
|         var rangeExpression = Validation.TrimToNull(affected.VulnerableRange); | ||||
|  | ||||
|         if (fixedVersion is null && rangeExpression is null) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return new AffectedVersionRange( | ||||
|             rangeKind, | ||||
|             introducedVersion: null, | ||||
|             fixedVersion: fixedVersion, | ||||
|             lastAffectedVersion: null, | ||||
|             rangeExpression: rangeExpression, | ||||
|             provenance: CreateRangeProvenance(identifier, recordedAt), | ||||
|             primitives: new RangePrimitives( | ||||
|                 SemVer: null, | ||||
|                 Nevra: null, | ||||
|                 Evr: null, | ||||
|                 VendorExtensions: CreateVendorExtensions(ecosystem, packageName))); | ||||
|     } | ||||
|  | ||||
|     private static AdvisoryProvenance CreateRangeProvenance(string identifier, DateTimeOffset recordedAt) | ||||
|         => new( | ||||
|             GhsaConnectorPlugin.SourceName, | ||||
|             "affected-range", | ||||
|             identifier, | ||||
|             recordedAt, | ||||
|             new[] { ProvenanceFieldMasks.VersionRanges }); | ||||
|  | ||||
|     private static IReadOnlyDictionary<string, string> CreateVendorExtensions(string ecosystem, string packageName) | ||||
|         => new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) | ||||
|         { | ||||
|             ["ecosystem"] = ecosystem, | ||||
|             ["package"] = packageName, | ||||
|         }; | ||||
|  | ||||
|     private static string? BuildNormalizedNote(string identifier) | ||||
|     { | ||||
|         var trimmed = Validation.TrimToNull(identifier); | ||||
|         return trimmed is null ? null : $"ghsa:{trimmed}"; | ||||
|     } | ||||
|  | ||||
|     private static string? ResolveRangeExpression(string? parsedExpression, string? constraintExpression, string? fallbackExpression) | ||||
|     { | ||||
|         var parsed = Validation.TrimToNull(parsedExpression); | ||||
|         if (parsed is not null) | ||||
|         { | ||||
|             return parsed; | ||||
|         } | ||||
|  | ||||
|         var constraint = Validation.TrimToNull(constraintExpression); | ||||
|         if (constraint is not null) | ||||
|         { | ||||
|             return constraint; | ||||
|         } | ||||
|  | ||||
|         return Validation.TrimToNull(fallbackExpression); | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,111 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| internal static class GhsaRateLimitParser | ||||
| { | ||||
|     public static GhsaRateLimitSnapshot? TryParse(IReadOnlyDictionary<string, string>? headers, DateTimeOffset now, string phase) | ||||
|     { | ||||
|         if (headers is null || headers.Count == 0) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         string? resource = null; | ||||
|         long? limit = null; | ||||
|         long? remaining = null; | ||||
|         long? used = null; | ||||
|         DateTimeOffset? resetAt = null; | ||||
|         TimeSpan? resetAfter = null; | ||||
|         TimeSpan? retryAfter = null; | ||||
|         var hasData = false; | ||||
|  | ||||
|         if (TryGet(headers, "X-RateLimit-Resource", out var resourceValue) && !string.IsNullOrWhiteSpace(resourceValue)) | ||||
|         { | ||||
|             resource = resourceValue; | ||||
|             hasData = true; | ||||
|         } | ||||
|  | ||||
|         if (TryParseLong(headers, "X-RateLimit-Limit", out var limitValue)) | ||||
|         { | ||||
|             limit = limitValue; | ||||
|             hasData = true; | ||||
|         } | ||||
|  | ||||
|         if (TryParseLong(headers, "X-RateLimit-Remaining", out var remainingValue)) | ||||
|         { | ||||
|             remaining = remainingValue; | ||||
|             hasData = true; | ||||
|         } | ||||
|  | ||||
|         if (TryParseLong(headers, "X-RateLimit-Used", out var usedValue)) | ||||
|         { | ||||
|             used = usedValue; | ||||
|             hasData = true; | ||||
|         } | ||||
|  | ||||
|         if (TryParseLong(headers, "X-RateLimit-Reset", out var resetValue)) | ||||
|         { | ||||
|             resetAt = DateTimeOffset.FromUnixTimeSeconds(resetValue); | ||||
|             var delta = resetAt.Value - now; | ||||
|             if (delta > TimeSpan.Zero) | ||||
|             { | ||||
|                 resetAfter = delta; | ||||
|             } | ||||
|             hasData = true; | ||||
|         } | ||||
|  | ||||
|         if (TryGet(headers, "Retry-After", out var retryAfterValue) && !string.IsNullOrWhiteSpace(retryAfterValue)) | ||||
|         { | ||||
|             if (double.TryParse(retryAfterValue, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds) && seconds > 0) | ||||
|             { | ||||
|                 retryAfter = TimeSpan.FromSeconds(seconds); | ||||
|             } | ||||
|             else if (DateTimeOffset.TryParse(retryAfterValue, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var retryAfterDate)) | ||||
|             { | ||||
|                 var delta = retryAfterDate - now; | ||||
|                 if (delta > TimeSpan.Zero) | ||||
|                 { | ||||
|                     retryAfter = delta; | ||||
|                 } | ||||
|             } | ||||
|             hasData = true; | ||||
|         } | ||||
|  | ||||
|         if (!hasData) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return new GhsaRateLimitSnapshot(phase, resource, limit, remaining, used, resetAt, resetAfter, retryAfter); | ||||
|     } | ||||
|  | ||||
|     private static bool TryGet(IReadOnlyDictionary<string, string> headers, string key, out string value) | ||||
|     { | ||||
|         foreach (var pair in headers) | ||||
|         { | ||||
|             if (pair.Key.Equals(key, StringComparison.OrdinalIgnoreCase)) | ||||
|             { | ||||
|                 value = pair.Value; | ||||
|                 return true; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         value = string.Empty; | ||||
|         return false; | ||||
|     } | ||||
|  | ||||
|     private static bool TryParseLong(IReadOnlyDictionary<string, string> headers, string key, out long result) | ||||
|     { | ||||
|         result = 0; | ||||
|         if (TryGet(headers, key, out var value) && long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) | ||||
|         { | ||||
|             result = parsed; | ||||
|             return true; | ||||
|         } | ||||
|  | ||||
|         return false; | ||||
|     } | ||||
| } | ||||
| @@ -0,0 +1,23 @@ | ||||
| using System; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| internal readonly record struct GhsaRateLimitSnapshot( | ||||
|     string Phase, | ||||
|     string? Resource, | ||||
|     long? Limit, | ||||
|     long? Remaining, | ||||
|     long? Used, | ||||
|     DateTimeOffset? ResetAt, | ||||
|     TimeSpan? ResetAfter, | ||||
|     TimeSpan? RetryAfter) | ||||
| { | ||||
|     public bool HasData => | ||||
|         Limit.HasValue || | ||||
|         Remaining.HasValue || | ||||
|         Used.HasValue || | ||||
|         ResetAt.HasValue || | ||||
|         ResetAfter.HasValue || | ||||
|         RetryAfter.HasValue || | ||||
|         !string.IsNullOrEmpty(Resource); | ||||
| } | ||||
| @@ -0,0 +1,75 @@ | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| internal sealed record GhsaRecordDto | ||||
| { | ||||
|     public string GhsaId { get; init; } = string.Empty; | ||||
|  | ||||
|     public string? Summary { get; init; } | ||||
|  | ||||
|     public string? Description { get; init; } | ||||
|  | ||||
|     public string? Severity { get; init; } | ||||
|  | ||||
|     public DateTimeOffset? PublishedAt { get; init; } | ||||
|  | ||||
|     public DateTimeOffset? UpdatedAt { get; init; } | ||||
|  | ||||
|     public IReadOnlyList<string> Aliases { get; init; } = Array.Empty<string>(); | ||||
|  | ||||
|     public IReadOnlyList<GhsaReferenceDto> References { get; init; } = Array.Empty<GhsaReferenceDto>(); | ||||
|  | ||||
|     public IReadOnlyList<GhsaAffectedDto> Affected { get; init; } = Array.Empty<GhsaAffectedDto>(); | ||||
|  | ||||
|     public IReadOnlyList<GhsaCreditDto> Credits { get; init; } = Array.Empty<GhsaCreditDto>(); | ||||
|  | ||||
|     public IReadOnlyList<GhsaWeaknessDto> Cwes { get; init; } = Array.Empty<GhsaWeaknessDto>(); | ||||
|  | ||||
|     public GhsaCvssDto? Cvss { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed record GhsaReferenceDto | ||||
| { | ||||
|     public string Url { get; init; } = string.Empty; | ||||
|  | ||||
|     public string? Type { get; init; } | ||||
|  | ||||
|     public string? Name { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed record GhsaAffectedDto | ||||
| { | ||||
|     public string PackageName { get; init; } = string.Empty; | ||||
|  | ||||
|     public string Ecosystem { get; init; } = string.Empty; | ||||
|  | ||||
|     public string? VulnerableRange { get; init; } | ||||
|  | ||||
|     public string? PatchedVersion { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed record GhsaCreditDto | ||||
| { | ||||
|     public string? Type { get; init; } | ||||
|  | ||||
|     public string? Name { get; init; } | ||||
|  | ||||
|     public string? Login { get; init; } | ||||
|  | ||||
|     public string? ProfileUrl { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed record GhsaWeaknessDto | ||||
| { | ||||
|     public string? CweId { get; init; } | ||||
|  | ||||
|     public string? Name { get; init; } | ||||
| } | ||||
|  | ||||
| internal sealed record GhsaCvssDto | ||||
| { | ||||
|     public double? Score { get; init; } | ||||
|  | ||||
|     public string? VectorString { get; init; } | ||||
|  | ||||
|     public string? Severity { get; init; } | ||||
| } | ||||
| @@ -0,0 +1,269 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.Globalization; | ||||
| using System.Text.Json; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa.Internal; | ||||
|  | ||||
| internal static class GhsaRecordParser | ||||
| { | ||||
|     public static GhsaRecordDto Parse(ReadOnlySpan<byte> content) | ||||
|     { | ||||
|         using var document = JsonDocument.Parse(content.ToArray()); | ||||
|         var root = document.RootElement; | ||||
|  | ||||
|         var ghsaId = GetString(root, "ghsa_id") ?? throw new JsonException("ghsa_id missing"); | ||||
|         var summary = GetString(root, "summary"); | ||||
|         var description = GetString(root, "description"); | ||||
|         var severity = GetString(root, "severity"); | ||||
|         var publishedAt = GetDate(root, "published_at"); | ||||
|         var updatedAt = GetDate(root, "updated_at") ?? publishedAt; | ||||
|  | ||||
|         var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) | ||||
|         { | ||||
|             ghsaId, | ||||
|         }; | ||||
|  | ||||
|         if (root.TryGetProperty("cve_ids", out var cveIds) && cveIds.ValueKind == JsonValueKind.Array) | ||||
|         { | ||||
|             foreach (var cve in cveIds.EnumerateArray()) | ||||
|             { | ||||
|                 if (cve.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(cve.GetString())) | ||||
|                 { | ||||
|                     aliases.Add(cve.GetString()!); | ||||
|                 } | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var references = ParseReferences(root); | ||||
|         var affected = ParseAffected(root); | ||||
|         var credits = ParseCredits(root); | ||||
|         var cwes = ParseCwes(root); | ||||
|         var cvss = ParseCvss(root); | ||||
|  | ||||
|         return new GhsaRecordDto | ||||
|         { | ||||
|             GhsaId = ghsaId, | ||||
|             Summary = summary, | ||||
|             Description = description, | ||||
|             Severity = severity, | ||||
|             PublishedAt = publishedAt, | ||||
|             UpdatedAt = updatedAt, | ||||
|             Aliases = aliases.ToArray(), | ||||
|             References = references, | ||||
|             Affected = affected, | ||||
|             Credits = credits, | ||||
|             Cwes = cwes, | ||||
|             Cvss = cvss, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<GhsaReferenceDto> ParseReferences(JsonElement root) | ||||
|     { | ||||
|         if (!root.TryGetProperty("references", out var references) || references.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return Array.Empty<GhsaReferenceDto>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<GhsaReferenceDto>(references.GetArrayLength()); | ||||
|         foreach (var reference in references.EnumerateArray()) | ||||
|         { | ||||
|             if (reference.ValueKind != JsonValueKind.Object) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var url = GetString(reference, "url"); | ||||
|             if (string.IsNullOrWhiteSpace(url)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             list.Add(new GhsaReferenceDto | ||||
|             { | ||||
|                 Url = url, | ||||
|                 Type = GetString(reference, "type"), | ||||
|                 Name = GetString(reference, "name"), | ||||
|             }); | ||||
|         } | ||||
|  | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<GhsaAffectedDto> ParseAffected(JsonElement root) | ||||
|     { | ||||
|         if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities) || vulnerabilities.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return Array.Empty<GhsaAffectedDto>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<GhsaAffectedDto>(vulnerabilities.GetArrayLength()); | ||||
|         foreach (var entry in vulnerabilities.EnumerateArray()) | ||||
|         { | ||||
|             if (entry.ValueKind != JsonValueKind.Object) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var package = entry.TryGetProperty("package", out var packageElement) && packageElement.ValueKind == JsonValueKind.Object | ||||
|                 ? packageElement | ||||
|                 : default; | ||||
|  | ||||
|             var packageName = GetString(package, "name") ?? "unknown-package"; | ||||
|             var ecosystem = GetString(package, "ecosystem") ?? "unknown"; | ||||
|             var vulnerableRange = GetString(entry, "vulnerable_version_range"); | ||||
|  | ||||
|             string? patchedVersion = null; | ||||
|             if (entry.TryGetProperty("first_patched_version", out var patchedElement) && patchedElement.ValueKind == JsonValueKind.Object) | ||||
|             { | ||||
|                 patchedVersion = GetString(patchedElement, "identifier"); | ||||
|             } | ||||
|  | ||||
|             list.Add(new GhsaAffectedDto | ||||
|             { | ||||
|                 PackageName = packageName, | ||||
|                 Ecosystem = ecosystem, | ||||
|                 VulnerableRange = vulnerableRange, | ||||
|                 PatchedVersion = patchedVersion, | ||||
|             }); | ||||
|         } | ||||
|  | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<GhsaCreditDto> ParseCredits(JsonElement root) | ||||
|     { | ||||
|         if (!root.TryGetProperty("credits", out var credits) || credits.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return Array.Empty<GhsaCreditDto>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<GhsaCreditDto>(credits.GetArrayLength()); | ||||
|         foreach (var credit in credits.EnumerateArray()) | ||||
|         { | ||||
|             if (credit.ValueKind != JsonValueKind.Object) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var type = GetString(credit, "type"); | ||||
|             var name = GetString(credit, "name"); | ||||
|             string? login = null; | ||||
|             string? profile = null; | ||||
|  | ||||
|             if (credit.TryGetProperty("user", out var user) && user.ValueKind == JsonValueKind.Object) | ||||
|             { | ||||
|                 login = GetString(user, "login"); | ||||
|                 profile = GetString(user, "html_url") ?? GetString(user, "url"); | ||||
|                 name ??= GetString(user, "name"); | ||||
|             } | ||||
|  | ||||
|             name ??= login; | ||||
|             if (string.IsNullOrWhiteSpace(name)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             list.Add(new GhsaCreditDto | ||||
|             { | ||||
|                 Type = type, | ||||
|                 Name = name, | ||||
|                 Login = login, | ||||
|                 ProfileUrl = profile, | ||||
|             }); | ||||
|         } | ||||
|  | ||||
|         return list; | ||||
|     } | ||||
|  | ||||
|     private static IReadOnlyList<GhsaWeaknessDto> ParseCwes(JsonElement root) | ||||
|     { | ||||
|         if (!root.TryGetProperty("cwes", out var cwes) || cwes.ValueKind != JsonValueKind.Array) | ||||
|         { | ||||
|             return Array.Empty<GhsaWeaknessDto>(); | ||||
|         } | ||||
|  | ||||
|         var list = new List<GhsaWeaknessDto>(cwes.GetArrayLength()); | ||||
|         foreach (var entry in cwes.EnumerateArray()) | ||||
|         { | ||||
|             if (entry.ValueKind != JsonValueKind.Object) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             var cweId = GetString(entry, "cwe_id"); | ||||
|             if (string.IsNullOrWhiteSpace(cweId)) | ||||
|             { | ||||
|                 continue; | ||||
|             } | ||||
|  | ||||
|             list.Add(new GhsaWeaknessDto | ||||
|             { | ||||
|                 CweId = cweId, | ||||
|                 Name = GetString(entry, "name"), | ||||
|             }); | ||||
|         } | ||||
|  | ||||
|         return list.Count == 0 ? Array.Empty<GhsaWeaknessDto>() : list; | ||||
|     } | ||||
|  | ||||
|     private static GhsaCvssDto? ParseCvss(JsonElement root) | ||||
|     { | ||||
|         if (!root.TryGetProperty("cvss", out var cvss) || cvss.ValueKind != JsonValueKind.Object) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         double? score = null; | ||||
|         if (cvss.TryGetProperty("score", out var scoreElement) && scoreElement.ValueKind == JsonValueKind.Number) | ||||
|         { | ||||
|             score = scoreElement.GetDouble(); | ||||
|         } | ||||
|  | ||||
|         var vector = GetString(cvss, "vector_string") ?? GetString(cvss, "vectorString"); | ||||
|         var severity = GetString(cvss, "severity"); | ||||
|  | ||||
|         if (score is null && string.IsNullOrWhiteSpace(vector) && string.IsNullOrWhiteSpace(severity)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return new GhsaCvssDto | ||||
|         { | ||||
|             Score = score, | ||||
|             VectorString = vector, | ||||
|             Severity = severity, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static string? GetString(JsonElement element, string propertyName) | ||||
|     { | ||||
|         if (element.ValueKind != JsonValueKind.Object) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         if (!element.TryGetProperty(propertyName, out var property)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return property.ValueKind switch | ||||
|         { | ||||
|             JsonValueKind.String => property.GetString(), | ||||
|             _ => null, | ||||
|         }; | ||||
|     } | ||||
|  | ||||
|     private static DateTimeOffset? GetDate(JsonElement element, string propertyName) | ||||
|     { | ||||
|         var value = GetString(element, propertyName); | ||||
|         if (string.IsNullOrWhiteSpace(value)) | ||||
|         { | ||||
|             return null; | ||||
|         } | ||||
|  | ||||
|         return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) | ||||
|             ? parsed.ToUniversalTime() | ||||
|             : null; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										43
									
								
								src/StellaOps.Concelier.Connector.Ghsa/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										43
									
								
								src/StellaOps.Concelier.Connector.Ghsa/Jobs.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,43 @@ | ||||
| using StellaOps.Concelier.Core.Jobs; | ||||
|  | ||||
| namespace StellaOps.Concelier.Connector.Ghsa; | ||||
|  | ||||
| internal static class GhsaJobKinds | ||||
| { | ||||
|     public const string Fetch = "source:ghsa:fetch"; | ||||
|     public const string Parse = "source:ghsa:parse"; | ||||
|     public const string Map = "source:ghsa:map"; | ||||
| } | ||||
|  | ||||
| internal sealed class GhsaFetchJob : IJob | ||||
| { | ||||
|     private readonly GhsaConnector _connector; | ||||
|  | ||||
|     public GhsaFetchJob(GhsaConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.FetchAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class GhsaParseJob : IJob | ||||
| { | ||||
|     private readonly GhsaConnector _connector; | ||||
|  | ||||
|     public GhsaParseJob(GhsaConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.ParseAsync(context.Services, cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class GhsaMapJob : IJob | ||||
| { | ||||
|     private readonly GhsaConnector _connector; | ||||
|  | ||||
|     public GhsaMapJob(GhsaConnector connector) | ||||
|         => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); | ||||
|  | ||||
|     public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) | ||||
|         => _connector.MapAsync(context.Services, cancellationToken); | ||||
| } | ||||
| @@ -0,0 +1,4 @@ | ||||
| using System.Runtime.CompilerServices; | ||||
|  | ||||
| [assembly: InternalsVisibleTo("FixtureUpdater")] | ||||
| [assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ghsa.Tests")] | ||||
| @@ -0,0 +1,17 @@ | ||||
| <Project Sdk="Microsoft.NET.Sdk"> | ||||
|  | ||||
|   <PropertyGroup> | ||||
|     <TargetFramework>net10.0</TargetFramework> | ||||
|     <ImplicitUsings>enable</ImplicitUsings> | ||||
|     <Nullable>enable</Nullable> | ||||
|   </PropertyGroup> | ||||
|  | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> | ||||
|  | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> | ||||
|     <ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" /> | ||||
|   </ItemGroup> | ||||
| </Project> | ||||
|  | ||||
							
								
								
									
										19
									
								
								src/StellaOps.Concelier.Connector.Ghsa/TASKS.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										19
									
								
								src/StellaOps.Concelier.Connector.Ghsa/TASKS.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,19 @@ | ||||
| # TASKS | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |Select GHSA data source & auth model|BE-Conn-GHSA|Research|**DONE (2025-10-10)** – Adopted GitHub Security Advisories REST (global) endpoint with bearer token + API version headers documented in `GhsaOptions`.| | ||||
| |Fetch pipeline & state management|BE-Conn-GHSA|Source.Common, Storage.Mongo|**DONE (2025-10-10)** – Implemented list/detail fetch using `GhsaCursor` (time window + page), resumable SourceState and backoff controls.| | ||||
| |DTO & parser implementation|BE-Conn-GHSA|Source.Common|**DONE (2025-10-10)** – Added `GhsaRecordParser`/DTOs extracting aliases, references, severity, vulnerable ranges, patched versions.| | ||||
| |Canonical mapping & range primitives|BE-Conn-GHSA|Models|**DONE (2025-10-10)** – `GhsaMapper` emits GHSA advisories with SemVer packages, vendor extensions (ecosystem/package) and deterministic references.<br>2025-10-11 research trail: upcoming normalized array should follow `[{"scheme":"semver","type":"range","min":"<min>","minInclusive":true,"max":"<max>","maxInclusive":false,"notes":"ghsa:GHSA-xxxx"}]`; include patched-only advisories as `lt`/`lte` when no explicit floor.| | ||||
| |Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-10)** – New `StellaOps.Concelier.Connector.Ghsa.Tests` regression covers fetch/parse/map via canned GHSA fixtures and snapshot assertions.| | ||||
| |Telemetry & documentation|DevEx|Docs|**DONE (2025-10-10)** – Diagnostics meter (`ghsa.fetch.*`) wired; DI extension documents token/headers and job registrations.| | ||||
| |GitHub quota monitoring & retries|BE-Conn-GHSA, Observability|Source.Common|**DONE (2025-10-12)** – Rate-limit metrics/logs added, retry/backoff handles 403 secondary limits, and ops runbook documents dashboards + mitigation steps.| | ||||
| |Production credential & scheduler rollout|Ops, BE-Conn-GHSA|Docs, WebService|**DONE (2025-10-12)** – Scheduler defaults registered via `JobSchedulerBuilder`, credential provisioning documented (Compose/Helm samples), and staged backfill guidance captured in `docs/ops/concelier-ghsa-operations.md`.| | ||||
| |FEEDCONN-GHSA-04-002 Conflict regression fixtures|BE-Conn-GHSA, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** – Added `conflict-ghsa.canonical.json` + `GhsaConflictFixtureTests`; SemVer ranges and credits align with merge precedence triple and shareable with QA. Validation: `dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj --filter GhsaConflictFixtureTests`.| | ||||
| |FEEDCONN-GHSA-02-004 GHSA credits & ecosystem severity mapping|BE-Conn-GHSA|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** – Mapper emits advisory credits with provenance masks, fixtures assert role/contact ordering, and severity normalization remains unchanged.| | ||||
| |FEEDCONN-GHSA-02-007 Credit parity regression fixtures|BE-Conn-GHSA, QA|Source.Nvd, Source.Osv|**DONE (2025-10-12)** – Parity fixtures regenerated via `tools/FixtureUpdater`, normalized SemVer notes verified against GHSA/NVD/OSV snapshots, and the fixtures guide now documents the headroom checks.| | ||||
| |FEEDCONN-GHSA-02-001 Normalized versions rollout|BE-Conn-GHSA|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-11)** – GHSA mapper now emits SemVer primitives + normalized ranges, fixtures refreshed, connector tests passing; report logged via FEEDMERGE-COORD-02-900.| | ||||
| |FEEDCONN-GHSA-02-005 Quota monitoring hardening|BE-Conn-GHSA, Observability|Source.Common metrics|**DONE (2025-10-12)** – Diagnostics expose headroom histograms/gauges, warning logs dedupe below the configured threshold, and the ops runbook gained alerting and mitigation guidance.| | ||||
| |FEEDCONN-GHSA-02-006 Scheduler rollout integration|BE-Conn-GHSA, Ops|Job scheduler|**DONE (2025-10-12)** – Dependency routine tests assert cron/timeouts, and the runbook highlights cron overrides plus backoff toggles for staged rollouts.| | ||||
| |FEEDCONN-GHSA-04-003 Description/CWE/metric parity rollout|BE-Conn-GHSA|Models, Core|**DONE (2025-10-15)** – Mapper emits advisory description, CWE weaknesses, and canonical CVSS metric id with updated fixtures (`osv-ghsa.osv.json` parity suite) and connector regression covers the new fields. Reported completion to Merge coordination.| | ||||
| |FEEDCONN-GHSA-04-004 Canonical metric fallback coverage|BE-Conn-GHSA|Models, Merge|**DONE (2025-10-16)** – Ensure canonical metric ids remain populated when GitHub omits CVSS vectors/scores; add fixtures capturing severity-only advisories, document precedence with Merge, and emit analytics to track fallback usage.<br>2025-10-16: Mapper now emits `ghsa:severity/<level>` canonical ids when vectors are missing, diagnostics expose `ghsa.map.canonical_metric_fallbacks`, conflict/mapper fixtures updated, and runbook documents Merge precedence. Tests: `dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj`.| | ||||
		Reference in New Issue
	
	Block a user