diff --git a/NuGet.config b/NuGet.config new file mode 100644 index 00000000..0e524b46 --- /dev/null +++ b/NuGet.config @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/local-nuget/Microsoft.Extensions.Http.Polly.9.0.0-preview.6.24328.4.nupkg b/local-nuget/Microsoft.Extensions.Http.Polly.9.0.0-preview.6.24328.4.nupkg new file mode 100644 index 00000000..290eb382 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Http.Polly.9.0.0-preview.6.24328.4.nupkg differ diff --git a/local-nuget/Mongo2Go.4.1.0.nupkg b/local-nuget/Mongo2Go.4.1.0.nupkg new file mode 100644 index 00000000..a9378128 Binary files /dev/null and b/local-nuget/Mongo2Go.4.1.0.nupkg differ diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs index a7017fef..c65e6867 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs @@ -64,12 +64,13 @@ public class StandardClientProvisioningStoreTests var result = await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None); Assert.True(result.Succeeded); - var document = Assert.Contains("signer", store.Documents); - Assert.Equal("attestor signer", document.Value.Properties[AuthorityClientMetadataKeys.Audiences]); + Assert.True(store.Documents.TryGetValue("signer", out var document)); + Assert.NotNull(document); + Assert.Equal("attestor signer", document!.Properties[AuthorityClientMetadataKeys.Audiences]); var descriptor = await provisioning.FindByClientIdAsync("signer", CancellationToken.None); Assert.NotNull(descriptor); - Assert.Equal(new[] { "attestor", "signer" }, descriptor!.Audiences.OrderBy(value => value, StringComparer.Ordinal)); + Assert.Equal(new[] { "attestor", "signer" }, descriptor!.AllowedAudiences.OrderBy(value => value, StringComparer.Ordinal)); } [Fact] @@ -101,8 +102,9 @@ public class StandardClientProvisioningStoreTests await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None); - var document = Assert.Contains("mtls-client", store.Documents).Value; - var binding = Assert.Single(document.CertificateBindings); + Assert.True(store.Documents.TryGetValue("mtls-client", out var document)); + Assert.NotNull(document); + var binding = Assert.Single(document!.CertificateBindings); Assert.Equal("AABBCCDD", binding.Thumbprint); Assert.Equal("01ff", binding.SerialNumber); Assert.Equal("CN=mtls-client", binding.Subject); diff --git a/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs b/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs index 1bcfb431..ff64b999 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs @@ -54,11 +54,11 @@ internal sealed record BootstrapClientRequest public IReadOnlyCollection? CertificateBindings { get; init; } } - -internal sealed record BootstrapInviteRequest -{ - public string Type { get; init; } = BootstrapInviteTypes.User; - + +internal sealed record BootstrapInviteRequest +{ + public string Type { get; init; } = BootstrapInviteTypes.User; + public string? Token { get; init; } public string? Provider { get; init; } @@ -91,31 +91,8 @@ internal sealed record BootstrapClientCertificateBinding public string? Label { get; init; } } -internal static class BootstrapInviteTypes -{ - public const string User = "user"; - public const string Client = "client"; -} - -internal sealed record BootstrapInviteRequest -{ - public string Type { get; init; } = BootstrapInviteTypes.User; - - public string? Token { get; init; } - - public string? Provider { get; init; } - - public string? Target { get; init; } - - public DateTimeOffset? ExpiresAt { get; init; } - - public string? IssuedBy { get; init; } - - public IReadOnlyDictionary? Metadata { get; init; } -} - -internal static class BootstrapInviteTypes -{ - public const string User = "user"; - public const string Client = "client"; -} +internal static class BootstrapInviteTypes +{ + public const string User = "user"; + public const string Client = "client"; +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Configuration/CertBundOptions.cs b/src/StellaOps.Concelier.Connector.CertBund/Configuration/CertBundOptions.cs new file mode 100644 index 00000000..f2fc1082 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Configuration/CertBundOptions.cs @@ -0,0 +1,104 @@ +using System.Net; + +namespace StellaOps.Concelier.Connector.CertBund.Configuration; + +public sealed class CertBundOptions +{ + public const string HttpClientName = "concelier.source.certbund"; + + /// + /// RSS feed providing the latest CERT-Bund advisories. + /// + public Uri FeedUri { get; set; } = new("https://wid.cert-bund.de/content/public/securityAdvisory/rss"); + + /// + /// Portal endpoint used to bootstrap session cookies (required for the SPA JSON API). + /// + public Uri PortalBootstrapUri { get; set; } = new("https://wid.cert-bund.de/portal/"); + + /// + /// Detail API endpoint template; advisory identifier is appended as the name query parameter. + /// + public Uri DetailApiUri { get; set; } = new("https://wid.cert-bund.de/portal/api/securityadvisory"); + + /// + /// Optional timeout override for feed/detail requests. + /// + public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Delay applied between successive detail fetches to respect upstream politeness. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + /// + /// Backoff recorded in source state when a fetch attempt fails. + /// + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Maximum number of advisories to enqueue per fetch iteration. + /// + public int MaxAdvisoriesPerFetch { get; set; } = 50; + + /// + /// Maximum number of advisory identifiers remembered to prevent re-processing. + /// + public int MaxKnownAdvisories { get; set; } = 512; + + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new InvalidOperationException("CERT-Bund feed URI must be an absolute URI."); + } + + if (PortalBootstrapUri is null || !PortalBootstrapUri.IsAbsoluteUri) + { + throw new InvalidOperationException("CERT-Bund portal bootstrap URI must be an absolute URI."); + } + + if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri) + { + throw new InvalidOperationException("CERT-Bund detail API URI must be an absolute URI."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero."); + } + + if (MaxKnownAdvisories <= 0) + { + throw new InvalidOperationException($"{nameof(MaxKnownAdvisories)} must be greater than zero."); + } + } + + public Uri BuildDetailUri(string advisoryId) + { + if (string.IsNullOrWhiteSpace(advisoryId)) + { + throw new ArgumentException("Advisory identifier must be provided.", nameof(advisoryId)); + } + + var builder = new UriBuilder(DetailApiUri); + var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; + builder.Query = $"{queryPrefix}name={Uri.EscapeDataString(advisoryId)}"; + return builder.Uri; + } +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundAdvisoryDto.cs new file mode 100644 index 00000000..6ec62c7e --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundAdvisoryDto.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +public sealed record CertBundAdvisoryDto +{ + [JsonPropertyName("advisoryId")] + public string AdvisoryId { get; init; } = string.Empty; + + [JsonPropertyName("title")] + public string Title { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("contentHtml")] + public string ContentHtml { get; init; } = string.Empty; + + [JsonPropertyName("severity")] + public string? Severity { get; init; } + + [JsonPropertyName("language")] + public string Language { get; init; } = "de"; + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("portalUri")] + public Uri PortalUri { get; init; } = new("https://wid.cert-bund.de/"); + + [JsonPropertyName("detailUri")] + public Uri DetailUri { get; init; } = new("https://wid.cert-bund.de/"); + + [JsonPropertyName("cveIds")] + public IReadOnlyList CveIds { get; init; } = Array.Empty(); + + [JsonPropertyName("products")] + public IReadOnlyList Products { get; init; } = Array.Empty(); + + [JsonPropertyName("references")] + public IReadOnlyList References { get; init; } = Array.Empty(); +} + +public sealed record CertBundProductDto +{ + [JsonPropertyName("vendor")] + public string? Vendor { get; init; } + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("versions")] + public string? Versions { get; init; } +} + +public sealed record CertBundReferenceDto +{ + [JsonPropertyName("url")] + public string Url { get; init; } = string.Empty; + + [JsonPropertyName("label")] + public string? Label { get; init; } +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundCursor.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundCursor.cs new file mode 100644 index 00000000..2bd39ad5 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundCursor.cs @@ -0,0 +1,118 @@ +using System; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +internal sealed record CertBundCursor( + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyCollection KnownAdvisories, + DateTimeOffset? LastPublished, + DateTimeOffset? LastFetchAt) +{ + private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); + private static readonly IReadOnlyCollection EmptyStrings = Array.Empty(); + + public static CertBundCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null); + + public CertBundCursor WithPendingDocuments(IEnumerable documents) + => this with { PendingDocuments = Distinct(documents) }; + + public CertBundCursor WithPendingMappings(IEnumerable mappings) + => this with { PendingMappings = Distinct(mappings) }; + + public CertBundCursor WithKnownAdvisories(IEnumerable advisories) + => this with { KnownAdvisories = advisories?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings }; + + public CertBundCursor WithLastPublished(DateTimeOffset? published) + => this with { LastPublished = published }; + + public CertBundCursor WithLastFetch(DateTimeOffset? timestamp) + => this with { LastFetchAt = timestamp }; + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + ["knownAdvisories"] = new BsonArray(KnownAdvisories), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + if (LastFetchAt.HasValue) + { + document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime; + } + + return document; + } + + public static CertBundCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var knownAdvisories = ReadStringArray(document, "knownAdvisories"); + var lastPublished = document.TryGetValue("lastPublished", out var publishedValue) + ? ParseDate(publishedValue) + : null; + var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue) + ? ParseDate(fetchValue) + : null; + + return new CertBundCursor(pendingDocuments, pendingMappings, knownAdvisories, lastPublished, lastFetch); + } + + private static IReadOnlyCollection Distinct(IEnumerable? values) + => values?.Distinct().ToArray() ?? EmptyGuids; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuids; + } + + var items = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element?.ToString(), out var id)) + { + items.Add(id); + } + } + + return items; + } + + private static IReadOnlyCollection ReadStringArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyStrings; + } + + return array.Select(element => element?.ToString() ?? string.Empty) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDetailParser.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDetailParser.cs new file mode 100644 index 00000000..0f713fdb --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDetailParser.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using StellaOps.Concelier.Connector.Common.Html; + +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +public sealed class CertBundDetailParser +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly HtmlContentSanitizer _sanitizer; + + public CertBundDetailParser(HtmlContentSanitizer sanitizer) + => _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer)); + + public CertBundAdvisoryDto Parse(Uri detailUri, Uri portalUri, byte[] payload) + { + var detail = JsonSerializer.Deserialize(payload, SerializerOptions) + ?? throw new InvalidOperationException("CERT-Bund detail payload deserialized to null."); + + var advisoryId = detail.Name ?? throw new InvalidOperationException("CERT-Bund detail missing advisory name."); + var contentHtml = _sanitizer.Sanitize(detail.Description ?? string.Empty, portalUri); + + return new CertBundAdvisoryDto + { + AdvisoryId = advisoryId, + Title = detail.Title ?? advisoryId, + Summary = detail.Summary, + ContentHtml = contentHtml, + Severity = detail.Severity, + Language = string.IsNullOrWhiteSpace(detail.Language) ? "de" : detail.Language!, + Published = detail.Published, + Modified = detail.Updated ?? detail.Published, + PortalUri = portalUri, + DetailUri = detailUri, + CveIds = detail.CveIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id!.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? Array.Empty(), + References = MapReferences(detail.References), + Products = MapProducts(detail.Products), + }; + } + + private static IReadOnlyList MapReferences(CertBundDetailReference[]? references) + { + if (references is null || references.Length == 0) + { + return Array.Empty(); + } + + return references + .Where(static reference => !string.IsNullOrWhiteSpace(reference.Url)) + .Select(reference => new CertBundReferenceDto + { + Url = reference.Url!, + Label = reference.Label, + }) + .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList MapProducts(CertBundDetailProduct[]? products) + { + if (products is null || products.Length == 0) + { + return Array.Empty(); + } + + return products + .Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name)) + .Select(product => new CertBundProductDto + { + Vendor = product.Vendor, + Name = product.Name, + Versions = product.Versions, + }) + .ToArray(); + } +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDetailResponse.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDetailResponse.cs new file mode 100644 index 00000000..92676484 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDetailResponse.cs @@ -0,0 +1,60 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +internal sealed record CertBundDetailResponse +{ + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("severity")] + public string? Severity { get; init; } + + [JsonPropertyName("language")] + public string? Language { get; init; } + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("updated")] + public DateTimeOffset? Updated { get; init; } + + [JsonPropertyName("cveIds")] + public string[]? CveIds { get; init; } + + [JsonPropertyName("references")] + public CertBundDetailReference[]? References { get; init; } + + [JsonPropertyName("products")] + public CertBundDetailProduct[]? Products { get; init; } +} + +internal sealed record CertBundDetailReference +{ + [JsonPropertyName("url")] + public string? Url { get; init; } + + [JsonPropertyName("label")] + public string? Label { get; init; } +} + +internal sealed record CertBundDetailProduct +{ + [JsonPropertyName("vendor")] + public string? Vendor { get; init; } + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("versions")] + public string? Versions { get; init; } +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDiagnostics.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDiagnostics.cs new file mode 100644 index 00000000..03c0cf68 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDiagnostics.cs @@ -0,0 +1,191 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +/// +/// Emits OpenTelemetry counters and histograms for the CERT-Bund connector. +/// +public sealed class CertBundDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Concelier.Connector.CertBund"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _feedFetchAttempts; + private readonly Counter _feedFetchSuccess; + private readonly Counter _feedFetchFailures; + private readonly Histogram _feedItemCount; + private readonly Histogram _feedEnqueuedCount; + private readonly Histogram _feedCoverageDays; + private readonly Counter _detailFetchAttempts; + private readonly Counter _detailFetchSuccess; + private readonly Counter _detailFetchNotModified; + private readonly Counter _detailFetchFailures; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Histogram _parseProductCount; + private readonly Histogram _parseCveCount; + private readonly Counter _mapSuccess; + private readonly Counter _mapFailures; + private readonly Histogram _mapPackageCount; + private readonly Histogram _mapAliasCount; + + public CertBundDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _feedFetchAttempts = _meter.CreateCounter( + name: "certbund.feed.fetch.attempts", + unit: "operations", + description: "Number of RSS feed load attempts."); + _feedFetchSuccess = _meter.CreateCounter( + name: "certbund.feed.fetch.success", + unit: "operations", + description: "Number of successful RSS feed loads."); + _feedFetchFailures = _meter.CreateCounter( + name: "certbund.feed.fetch.failures", + unit: "operations", + description: "Number of RSS feed load failures."); + _feedItemCount = _meter.CreateHistogram( + name: "certbund.feed.items.count", + unit: "items", + description: "Distribution of RSS item counts per fetch."); + _feedEnqueuedCount = _meter.CreateHistogram( + name: "certbund.feed.enqueued.count", + unit: "documents", + description: "Distribution of advisory documents enqueued per fetch."); + _feedCoverageDays = _meter.CreateHistogram( + name: "certbund.feed.coverage.days", + unit: "days", + description: "Coverage window in days between fetch time and the oldest published advisory in the feed."); + _detailFetchAttempts = _meter.CreateCounter( + name: "certbund.detail.fetch.attempts", + unit: "operations", + description: "Number of detail fetch attempts."); + _detailFetchSuccess = _meter.CreateCounter( + name: "certbund.detail.fetch.success", + unit: "operations", + description: "Number of detail fetches that persisted a document."); + _detailFetchNotModified = _meter.CreateCounter( + name: "certbund.detail.fetch.not_modified", + unit: "operations", + description: "Number of detail fetches returning HTTP 304."); + _detailFetchFailures = _meter.CreateCounter( + name: "certbund.detail.fetch.failures", + unit: "operations", + description: "Number of detail fetches that failed."); + _parseSuccess = _meter.CreateCounter( + name: "certbund.parse.success", + unit: "documents", + description: "Number of documents parsed into CERT-Bund DTOs."); + _parseFailures = _meter.CreateCounter( + name: "certbund.parse.failures", + unit: "documents", + description: "Number of documents that failed to parse."); + _parseProductCount = _meter.CreateHistogram( + name: "certbund.parse.products.count", + unit: "products", + description: "Distribution of product entries captured per advisory."); + _parseCveCount = _meter.CreateHistogram( + name: "certbund.parse.cve.count", + unit: "aliases", + description: "Distribution of CVE identifiers captured per advisory."); + _mapSuccess = _meter.CreateCounter( + name: "certbund.map.success", + unit: "advisories", + description: "Number of canonical advisories emitted by the mapper."); + _mapFailures = _meter.CreateCounter( + name: "certbund.map.failures", + unit: "advisories", + description: "Number of mapping failures."); + _mapPackageCount = _meter.CreateHistogram( + name: "certbund.map.affected.count", + unit: "packages", + description: "Distribution of affected packages emitted per advisory."); + _mapAliasCount = _meter.CreateHistogram( + name: "certbund.map.aliases.count", + unit: "aliases", + description: "Distribution of alias counts per advisory."); + } + + public void FeedFetchAttempt() => _feedFetchAttempts.Add(1); + + public void FeedFetchSuccess(int itemCount) + { + _feedFetchSuccess.Add(1); + if (itemCount >= 0) + { + _feedItemCount.Record(itemCount); + } + } + + public void FeedFetchFailure(string reason = "error") + => _feedFetchFailures.Add(1, ReasonTag(reason)); + + public void RecordFeedCoverage(double? coverageDays) + { + if (coverageDays is { } days && days >= 0) + { + _feedCoverageDays.Record(days); + } + } + + public void DetailFetchAttempt() => _detailFetchAttempts.Add(1); + + public void DetailFetchSuccess() => _detailFetchSuccess.Add(1); + + public void DetailFetchNotModified() => _detailFetchNotModified.Add(1); + + public void DetailFetchFailure(string reason = "error") + => _detailFetchFailures.Add(1, ReasonTag(reason)); + + public void DetailFetchEnqueued(int count) + { + if (count >= 0) + { + _feedEnqueuedCount.Record(count); + } + } + + public void ParseSuccess(int productCount, int cveCount) + { + _parseSuccess.Add(1); + + if (productCount >= 0) + { + _parseProductCount.Record(productCount); + } + + if (cveCount >= 0) + { + _parseCveCount.Record(cveCount); + } + } + + public void ParseFailure(string reason = "error") + => _parseFailures.Add(1, ReasonTag(reason)); + + public void MapSuccess(int affectedPackages, int aliasCount) + { + _mapSuccess.Add(1); + + if (affectedPackages >= 0) + { + _mapPackageCount.Record(affectedPackages); + } + + if (aliasCount >= 0) + { + _mapAliasCount.Record(aliasCount); + } + } + + public void MapFailure(string reason = "error") + => _mapFailures.Add(1, ReasonTag(reason)); + + private static KeyValuePair ReasonTag(string reason) + => new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant()); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDocumentMetadata.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDocumentMetadata.cs new file mode 100644 index 00000000..26149696 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundDocumentMetadata.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +internal static class CertBundDocumentMetadata +{ + public static Dictionary CreateMetadata(CertBundFeedItem item) + { + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["certbund.advisoryId"] = item.AdvisoryId, + ["certbund.portalUri"] = item.PortalUri.ToString(), + ["certbund.published"] = item.Published.ToString("O"), + }; + + if (!string.IsNullOrWhiteSpace(item.Category)) + { + metadata["certbund.category"] = item.Category!; + } + + if (!string.IsNullOrWhiteSpace(item.Title)) + { + metadata["certbund.title"] = item.Title!; + } + + return metadata; + } +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundFeedClient.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundFeedClient.cs new file mode 100644 index 00000000..9641e051 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundFeedClient.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.CertBund.Configuration; + +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +public sealed class CertBundFeedClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly CertBundOptions _options; + private readonly ILogger _logger; + private readonly SemaphoreSlim _bootstrapSemaphore = new(1, 1); + private volatile bool _bootstrapped; + + public CertBundFeedClient( + IHttpClientFactory httpClientFactory, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> LoadAsync(CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(CertBundOptions.HttpClientName); + await EnsureSessionAsync(client, cancellationToken).ConfigureAwait(false); + + using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri); + request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8"); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var document = XDocument.Load(stream); + + var items = new List(); + foreach (var element in document.Descendants("item")) + { + cancellationToken.ThrowIfCancellationRequested(); + + var linkValue = element.Element("link")?.Value?.Trim(); + if (string.IsNullOrWhiteSpace(linkValue) || !Uri.TryCreate(linkValue, UriKind.Absolute, out var portalUri)) + { + continue; + } + + var advisoryId = TryExtractNameParameter(portalUri); + if (string.IsNullOrWhiteSpace(advisoryId)) + { + continue; + } + + var detailUri = _options.BuildDetailUri(advisoryId); + var pubDateText = element.Element("pubDate")?.Value; + var published = ParseDate(pubDateText); + var title = element.Element("title")?.Value?.Trim(); + var category = element.Element("category")?.Value?.Trim(); + + items.Add(new CertBundFeedItem(advisoryId, detailUri, portalUri, published, title, category)); + } + + return items; + } + + private async Task EnsureSessionAsync(HttpClient client, CancellationToken cancellationToken) + { + if (_bootstrapped) + { + return; + } + + await _bootstrapSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_bootstrapped) + { + return; + } + + using var request = new HttpRequestMessage(HttpMethod.Get, _options.PortalBootstrapUri); + request.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + _bootstrapped = true; + } + finally + { + _bootstrapSemaphore.Release(); + } + } + + private static string? TryExtractNameParameter(Uri portalUri) + { + if (portalUri is null) + { + return null; + } + + var query = portalUri.Query; + if (string.IsNullOrEmpty(query)) + { + return null; + } + + var trimmed = query.TrimStart('?'); + foreach (var pair in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var separatorIndex = pair.IndexOf('='); + if (separatorIndex <= 0) + { + continue; + } + + var key = pair[..separatorIndex].Trim(); + if (!key.Equals("name", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var value = pair[(separatorIndex + 1)..]; + return Uri.UnescapeDataString(value); + } + + return null; + } + + private static DateTimeOffset ParseDate(string? value) + => DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) + ? parsed + : DateTimeOffset.UtcNow; +} diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundFeedItem.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundFeedItem.cs new file mode 100644 index 00000000..40eab8f0 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundFeedItem.cs @@ -0,0 +1,11 @@ +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +using System; + +public sealed record CertBundFeedItem( + string AdvisoryId, + Uri DetailUri, + Uri PortalUri, + DateTimeOffset Published, + string? Title, + string? Category); diff --git a/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundMapper.cs b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundMapper.cs new file mode 100644 index 00000000..540e9660 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertBund/Internal/CertBundMapper.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.CertBund.Internal; + +internal static class CertBundMapper +{ + public static Advisory Map(CertBundAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var packages = BuildPackages(dto, recordedAt); + var provenance = new AdvisoryProvenance( + CertBundConnectorPlugin.SourceName, + "advisory", + dto.AdvisoryId, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + + return new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title, + summary: dto.Summary, + language: dto.Language?.ToLowerInvariant() ?? "de", + published: dto.Published, + modified: dto.Modified, + severity: MapSeverity(dto.Severity), + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static IReadOnlyList BuildAliases(CertBundAdvisoryDto dto) + { + var aliases = new List(capacity: 4) { dto.AdvisoryId }; + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve); + } + } + + return aliases + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildReferences(CertBundAdvisoryDto dto, DateTimeOffset recordedAt) + { + var references = new List + { + new(dto.DetailUri.ToString(), "details", "cert-bund", null, new AdvisoryProvenance( + CertBundConnectorPlugin.SourceName, + "reference", + dto.DetailUri.ToString(), + recordedAt, + new[] { ProvenanceFieldMasks.References })) + }; + + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + references.Add(new AdvisoryReference( + reference.Url, + kind: "reference", + sourceTag: "cert-bund", + summary: reference.Label, + provenance: new AdvisoryProvenance( + CertBundConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt, + new[] { ProvenanceFieldMasks.References }))); + } + + return references + .DistinctBy(static reference => reference.Url, StringComparer.Ordinal) + .OrderBy(static reference => reference.Url, StringComparer.Ordinal) + .ToArray(); + } + + private static IReadOnlyList BuildPackages(CertBundAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.Products.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Products.Count); + foreach (var product in dto.Products) + { + var vendor = Validation.TrimToNull(product.Vendor) ?? "Unspecified"; + var name = Validation.TrimToNull(product.Name); + var identifier = name is null ? vendor : $"{vendor} {name}"; + + var provenance = new AdvisoryProvenance( + CertBundConnectorPlugin.SourceName, + "package", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }); + + var ranges = string.IsNullOrWhiteSpace(product.Versions) + ? Array.Empty() + : new[] + { + new AffectedVersionRange( + rangeKind: "string", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: product.Versions, + provenance: new AdvisoryProvenance( + CertBundConnectorPlugin.SourceName, + "package-range", + product.Versions, + recordedAt, + new[] { ProvenanceFieldMasks.VersionRanges })) + }; + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + platform: null, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: new[] { provenance }, + normalizedVersions: Array.Empty())); + } + + return packages + .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? MapSeverity(string? severity) + { + if (string.IsNullOrWhiteSpace(severity)) + { + return null; + } + + return severity.ToLowerInvariant() switch + { + "hoch" or "high" => "high", + "mittel" or "medium" => "medium", + "gering" or "low" => "low", + _ => severity.ToLowerInvariant(), + }; + } +} diff --git a/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs index 219e89ff..3ca3489c 100644 --- a/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs @@ -1,162 +1,187 @@ -using MongoDB.Bson; -using StellaOps.Concelier.Connector.Common.Cursors; - -namespace StellaOps.Concelier.Connector.CertCc.Internal; - -internal sealed record CertCcCursor( - TimeWindowCursorState SummaryState, - IReadOnlyCollection PendingSummaries, - IReadOnlyCollection PendingNotes, - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings, - DateTimeOffset? LastRun) -{ - private static readonly Guid[] EmptyGuidArray = Array.Empty(); - private static readonly string[] EmptyStringArray = Array.Empty(); - - public static CertCcCursor Empty { get; } = new( - TimeWindowCursorState.Empty, - EmptyGuidArray, - EmptyStringArray, - EmptyGuidArray, - EmptyGuidArray, - null); - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument(); - - var summary = new BsonDocument(); - SummaryState.WriteTo(summary, "start", "end"); - document["summary"] = summary; - - document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString())); - document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note)); - document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())); - document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())); - - if (LastRun.HasValue) - { - document["lastRun"] = LastRun.Value.UtcDateTime; - } - - return document; - } - - public static CertCcCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - TimeWindowCursorState summaryState = TimeWindowCursorState.Empty; - if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument) - { - summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end"); - } - - var pendingSummaries = ReadGuidArray(document, "pendingSummaries"); - var pendingNotes = ReadStringArray(document, "pendingNotes"); - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - - DateTimeOffset? lastRun = null; - if (document.TryGetValue("lastRun", out var lastRunValue)) - { - lastRun = lastRunValue.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - } - - return new CertCcCursor(summaryState, pendingSummaries, pendingNotes, pendingDocuments, pendingMappings, lastRun); - } - - public CertCcCursor WithSummaryState(TimeWindowCursorState state) - => this with { SummaryState = state ?? TimeWindowCursorState.Empty }; - - public CertCcCursor WithPendingSummaries(IEnumerable? ids) - => this with { PendingSummaries = NormalizeGuidSet(ids) }; - - public CertCcCursor WithPendingNotes(IEnumerable? notes) - => this with { PendingNotes = NormalizeStringSet(notes) }; - - public CertCcCursor WithPendingDocuments(IEnumerable? ids) - => this with { PendingDocuments = NormalizeGuidSet(ids) }; - - public CertCcCursor WithPendingMappings(IEnumerable? ids) - => this with { PendingMappings = NormalizeGuidSet(ids) }; - - public CertCcCursor WithLastRun(DateTimeOffset? timestamp) - => this with { LastRun = timestamp }; - - private static Guid[] ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) - { - return EmptyGuidArray; - } - - var results = new List(array.Count); - foreach (var element in array) - { - if (element is BsonString bsonString && Guid.TryParse(bsonString.AsString, out var parsed)) - { - results.Add(parsed); - continue; - } - - if (element is BsonBinaryData binary && binary.GuidRepresentation != GuidRepresentation.Unspecified) - { - results.Add(binary.ToGuid()); - } - } - - return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray(); - } - - private static string[] ReadStringArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) - { - return EmptyStringArray; - } - - var results = new List(array.Count); - foreach (var element in array) - { - switch (element) - { - case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString): - results.Add(bsonString.AsString.Trim()); - break; - case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString: - results.Add(inner.AsString.Trim()); - break; - } - } - - return results.Count == 0 - ? EmptyStringArray - : results - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Select(static value => value.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static Guid[] NormalizeGuidSet(IEnumerable? ids) - => ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray; - - private static string[] NormalizeStringSet(IEnumerable? values) - => values is null - ? EmptyStringArray - : values - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Select(static value => value.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); -} +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common.Cursors; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +internal sealed record CertCcCursor( + TimeWindowCursorState SummaryState, + IReadOnlyCollection PendingSummaries, + IReadOnlyCollection PendingNotes, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + DateTimeOffset? LastRun) +{ + private static readonly Guid[] EmptyGuidArray = Array.Empty(); + private static readonly string[] EmptyStringArray = Array.Empty(); + + public static CertCcCursor Empty { get; } = new( + TimeWindowCursorState.Empty, + EmptyGuidArray, + EmptyStringArray, + EmptyGuidArray, + EmptyGuidArray, + null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + + var summary = new BsonDocument(); + SummaryState.WriteTo(summary, "start", "end"); + document["summary"] = summary; + + document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString())); + document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note)); + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())); + + if (LastRun.HasValue) + { + document["lastRun"] = LastRun.Value.UtcDateTime; + } + + return document; + } + + public static CertCcCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + TimeWindowCursorState summaryState = TimeWindowCursorState.Empty; + if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument) + { + summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end"); + } + + var pendingSummaries = ReadGuidArray(document, "pendingSummaries"); + var pendingNotes = ReadStringArray(document, "pendingNotes"); + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + DateTimeOffset? lastRun = null; + if (document.TryGetValue("lastRun", out var lastRunValue)) + { + lastRun = lastRunValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new CertCcCursor(summaryState, pendingSummaries, pendingNotes, pendingDocuments, pendingMappings, lastRun); + } + + public CertCcCursor WithSummaryState(TimeWindowCursorState state) + => this with { SummaryState = state ?? TimeWindowCursorState.Empty }; + + public CertCcCursor WithPendingSummaries(IEnumerable? ids) + => this with { PendingSummaries = NormalizeGuidSet(ids) }; + + public CertCcCursor WithPendingNotes(IEnumerable? notes) + => this with { PendingNotes = NormalizeStringSet(notes) }; + + public CertCcCursor WithPendingDocuments(IEnumerable? ids) + => this with { PendingDocuments = NormalizeGuidSet(ids) }; + + public CertCcCursor WithPendingMappings(IEnumerable? ids) + => this with { PendingMappings = NormalizeGuidSet(ids) }; + + public CertCcCursor WithLastRun(DateTimeOffset? timestamp) + => this with { LastRun = timestamp }; + + private static Guid[] ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) + { + return EmptyGuidArray; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (TryReadGuid(element, out var parsed)) + { + results.Add(parsed); + } + } + + return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray(); + } + + private static string[] ReadStringArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) + { + return EmptyStringArray; + } + + var results = new List(array.Count); + foreach (var element in array) + { + switch (element) + { + case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString): + results.Add(bsonString.AsString.Trim()); + break; + case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString: + results.Add(inner.AsString.Trim()); + break; + } + } + + return results.Count == 0 + ? EmptyStringArray + : results + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static bool TryReadGuid(BsonValue value, out Guid guid) + { + if (value is BsonString bsonString && Guid.TryParse(bsonString.AsString, out guid)) + { + return true; + } + + if (value is BsonBinaryData binary) + { + try + { + guid = binary.ToGuid(); + return true; + } + catch (FormatException) + { + // ignore and fall back to byte array parsing + } + + var bytes = binary.AsByteArray; + if (bytes.Length == 16) + { + guid = new Guid(bytes); + return true; + } + } + + guid = default; + return false; + } + + private static Guid[] NormalizeGuidSet(IEnumerable? ids) + => ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray; + + private static string[] NormalizeStringSet(IEnumerable? values) + => values is null + ? EmptyStringArray + : values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Configuration/CiscoOptions.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Configuration/CiscoOptions.cs new file mode 100644 index 00000000..fd1768e3 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Configuration/CiscoOptions.cs @@ -0,0 +1,124 @@ +using System.Globalization; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; + +public sealed class CiscoOptions +{ + public const string HttpClientName = "concelier.source.vndr.cisco"; + public const string AuthHttpClientName = "concelier.source.vndr.cisco.auth"; + + public Uri BaseUri { get; set; } = new("https://api.cisco.com/security/advisories/v2/", UriKind.Absolute); + + public Uri TokenEndpoint { get; set; } = new("https://id.cisco.com/oauth2/default/v1/token", UriKind.Absolute); + + public string ClientId { get; set; } = string.Empty; + + public string ClientSecret { get; set; } = string.Empty; + + public int PageSize { get; set; } = 100; + + public int MaxPagesPerFetch { get; set; } = 5; + + public int MaxAdvisoriesPerFetch { get; set; } = 200; + + public TimeSpan InitialBackfillWindow { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); + + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); + + public TimeSpan TokenRefreshSkew { get; set; } = TimeSpan.FromMinutes(1); + + public string LastModifiedPathTemplate { get; set; } = "advisories/lastmodified/{0}"; + + public void Validate() + { + if (BaseUri is null || !BaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Cisco BaseUri must be an absolute URI."); + } + + if (TokenEndpoint is null || !TokenEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("Cisco TokenEndpoint must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(ClientId)) + { + throw new InvalidOperationException("Cisco clientId must be configured."); + } + + if (string.IsNullOrWhiteSpace(ClientSecret)) + { + throw new InvalidOperationException("Cisco clientSecret must be configured."); + } + + if (PageSize is < 1 or > 100) + { + throw new InvalidOperationException("Cisco PageSize must be between 1 and 100."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("Cisco MaxPagesPerFetch must be greater than zero."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException("Cisco MaxAdvisoriesPerFetch must be greater than zero."); + } + + if (InitialBackfillWindow <= TimeSpan.Zero) + { + throw new InvalidOperationException("Cisco InitialBackfillWindow must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("Cisco RequestDelay cannot be negative."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Cisco RequestTimeout must be positive."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException("Cisco FailureBackoff must be positive."); + } + + if (TokenRefreshSkew < TimeSpan.FromSeconds(5)) + { + throw new InvalidOperationException("Cisco TokenRefreshSkew must be at least 5 seconds."); + } + + if (string.IsNullOrWhiteSpace(LastModifiedPathTemplate)) + { + throw new InvalidOperationException("Cisco LastModifiedPathTemplate must be configured."); + } + } + + public Uri BuildLastModifiedUri(DateOnly date, int pageIndex, int pageSize) + { + if (pageIndex < 1) + { + throw new ArgumentOutOfRangeException(nameof(pageIndex), pageIndex, "Page index must be >= 1."); + } + + if (pageSize is < 1 or > 100) + { + throw new ArgumentOutOfRangeException(nameof(pageSize), pageSize, "Page size must be between 1 and 100."); + } + + var path = string.Format(CultureInfo.InvariantCulture, LastModifiedPathTemplate, date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); + var builder = new UriBuilder(BaseUri); + var basePath = builder.Path.TrimEnd('/'); + builder.Path = $"{basePath}/{path}".Replace("//", "/", StringComparison.Ordinal); + var query = $"pageIndex={pageIndex.ToString(CultureInfo.InvariantCulture)}&pageSize={pageSize.ToString(CultureInfo.InvariantCulture)}"; + builder.Query = string.IsNullOrEmpty(builder.Query) ? query : builder.Query.TrimStart('?') + "&" + query; + return builder.Uri; + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs new file mode 100644 index 00000000..86cecf49 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs @@ -0,0 +1,145 @@ +using System.Net.Http.Headers; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +internal sealed class CiscoAccessTokenProvider : IDisposable +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly SemaphoreSlim _refreshLock = new(1, 1); + + private volatile AccessToken? _cached; + private bool _disposed; + + public CiscoAccessTokenProvider( + IHttpClientFactory httpClientFactory, + IOptionsMonitor options, + TimeProvider? timeProvider, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task GetTokenAsync(CancellationToken cancellationToken) + => await GetTokenInternalAsync(forceRefresh: false, cancellationToken).ConfigureAwait(false); + + public void Invalidate() + => _cached = null; + + private async Task GetTokenInternalAsync(bool forceRefresh, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + + var options = _options.CurrentValue; + var now = _timeProvider.GetUtcNow(); + var cached = _cached; + if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew) + { + return cached.Value; + } + + await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + cached = _cached; + now = _timeProvider.GetUtcNow(); + if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew) + { + return cached.Value; + } + + var fresh = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false); + _cached = fresh; + return fresh.Value; + } + finally + { + _refreshLock.Release(); + } + } + + private async Task RequestTokenAsync(CiscoOptions options, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(CiscoOptions.AuthHttpClientName); + client.Timeout = options.RequestTimeout; + + using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint); + request.Headers.Accept.Clear(); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + var content = new FormUrlEncodedContent(new Dictionary + { + ["grant_type"] = "client_credentials", + ["client_id"] = options.ClientId, + ["client_secret"] = options.ClientSecret, + }); + + request.Content = content; + + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var message = $"Cisco OAuth token request failed with status {(int)response.StatusCode} {response.StatusCode}."; + _logger.LogError("Cisco openVuln token request failed: {Message}; response={Preview}", message, preview); + throw new HttpRequestException(message); + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var payload = await JsonSerializer.DeserializeAsync(stream, SerializerOptions, cancellationToken).ConfigureAwait(false); + if (payload is null || string.IsNullOrWhiteSpace(payload.AccessToken)) + { + throw new InvalidOperationException("Cisco OAuth token response did not include an access token."); + } + + var expiresIn = payload.ExpiresIn > 0 ? TimeSpan.FromSeconds(payload.ExpiresIn) : TimeSpan.FromHours(1); + var now = _timeProvider.GetUtcNow(); + var expiresAt = now + expiresIn; + _logger.LogInformation("Cisco openVuln token issued; expires in {ExpiresIn}", expiresIn); + return new AccessToken(payload.AccessToken, expiresAt); + } + + public async Task RefreshAsync(CancellationToken cancellationToken) + => await GetTokenInternalAsync(forceRefresh: true, cancellationToken).ConfigureAwait(false); + + private void ThrowIfDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(CiscoAccessTokenProvider)); + } + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _refreshLock.Dispose(); + _disposed = true; + } + + private sealed record AccessToken(string Value, DateTimeOffset ExpiresAt); + + private sealed record TokenResponse( + [property: JsonPropertyName("access_token")] string AccessToken, + [property: JsonPropertyName("expires_in")] int ExpiresIn, + [property: JsonPropertyName("token_type")] string? TokenType); +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAdvisoryDto.cs new file mode 100644 index 00000000..c5f14419 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoAdvisoryDto.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +public sealed record CiscoAdvisoryDto( + string AdvisoryId, + string Title, + string? Summary, + string? Severity, + DateTimeOffset? Published, + DateTimeOffset? Updated, + string? PublicationUrl, + string? CsafUrl, + string? CvrfUrl, + double? CvssBaseScore, + IReadOnlyList Cves, + IReadOnlyList BugIds, + IReadOnlyList Products); + +public sealed record CiscoAffectedProductDto( + string Name, + string? ProductId, + string? Version, + IReadOnlyCollection Statuses); diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafClient.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafClient.cs new file mode 100644 index 00000000..9f8d2870 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafClient.cs @@ -0,0 +1,64 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Text; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +public interface ICiscoCsafClient +{ + Task TryFetchAsync(string? url, CancellationToken cancellationToken); +} + +public class CiscoCsafClient : ICiscoCsafClient +{ + private static readonly string[] AcceptHeaders = { "application/json", "application/csaf+json", "application/vnd.cisco.csaf+json" }; + + private readonly SourceFetchService _fetchService; + private readonly ILogger _logger; + + public CiscoCsafClient(SourceFetchService fetchService, ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public virtual async Task TryFetchAsync(string? url, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(url)) + { + return null; + } + + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) + { + _logger.LogWarning("Cisco CSAF URL '{Url}' is not a valid absolute URI.", url); + return null; + } + + try + { + var request = new SourceFetchRequest(CiscoOptions.HttpClientName, VndrCiscoConnectorPlugin.SourceName, uri) + { + AcceptHeaders = AcceptHeaders, + }; + + var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Content is null) + { + _logger.LogWarning("Cisco CSAF download returned status {Status} for {Url}", result.StatusCode, url); + return null; + } + + return System.Text.Encoding.UTF8.GetString(result.Content); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or IOException or InvalidOperationException) + { + _logger.LogWarning(ex, "Cisco CSAF download failed for {Url}", url); + return null; + } + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafData.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafData.cs new file mode 100644 index 00000000..cc1e87d9 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafData.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +internal sealed record CiscoCsafData( + IReadOnlyDictionary Products, + IReadOnlyDictionary> ProductStatuses); + +internal sealed record CiscoCsafProduct(string ProductId, string Name); diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafParser.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafParser.cs new file mode 100644 index 00000000..be956e3e --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCsafParser.cs @@ -0,0 +1,123 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +internal static class CiscoCsafParser +{ + public static CiscoCsafData Parse(string content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return new CiscoCsafData( + Products: new Dictionary(0, StringComparer.OrdinalIgnoreCase), + ProductStatuses: new Dictionary>(0, StringComparer.OrdinalIgnoreCase)); + } + + using var document = JsonDocument.Parse(content); + var root = document.RootElement; + + var products = ParseProducts(root); + var statuses = ParseStatuses(root); + + return new CiscoCsafData(products, statuses); + } + + private static IReadOnlyDictionary ParseProducts(JsonElement root) + { + var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (!root.TryGetProperty("product_tree", out var productTree)) + { + return dictionary; + } + + if (productTree.TryGetProperty("full_product_names", out var fullProductNames) + && fullProductNames.ValueKind == JsonValueKind.Array) + { + foreach (var entry in fullProductNames.EnumerateArray()) + { + var productId = entry.TryGetProperty("product_id", out var idElement) && idElement.ValueKind == JsonValueKind.String + ? idElement.GetString() + : null; + + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + var name = entry.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String + ? nameElement.GetString() + : null; + + if (string.IsNullOrWhiteSpace(name)) + { + name = productId; + } + + dictionary[productId] = new CiscoCsafProduct(productId, name); + } + } + + return dictionary; + } + + private static IReadOnlyDictionary> ParseStatuses(JsonElement root) + { + var map = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities) + || vulnerabilities.ValueKind != JsonValueKind.Array) + { + return map.ToDictionary( + static kvp => kvp.Key, + static kvp => (IReadOnlyCollection)kvp.Value.ToArray(), + StringComparer.OrdinalIgnoreCase); + } + + foreach (var vulnerability in vulnerabilities.EnumerateArray()) + { + if (!vulnerability.TryGetProperty("product_status", out var productStatus) + || productStatus.ValueKind != JsonValueKind.Object) + { + continue; + } + + foreach (var property in productStatus.EnumerateObject()) + { + var statusLabel = property.Name; + if (property.Value.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var productIdElement in property.Value.EnumerateArray()) + { + if (productIdElement.ValueKind != JsonValueKind.String) + { + continue; + } + + var productId = productIdElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + if (!map.TryGetValue(productId, out var set)) + { + set = new HashSet(StringComparer.OrdinalIgnoreCase); + map[productId] = set; + } + + set.Add(statusLabel); + } + } + } + + return map.ToDictionary( + static kvp => kvp.Key, + static kvp => (IReadOnlyCollection)kvp.Value.ToArray(), + StringComparer.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCursor.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCursor.cs new file mode 100644 index 00000000..528cc28e --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoCursor.cs @@ -0,0 +1,101 @@ +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +internal sealed record CiscoCursor( + DateTimeOffset? LastModified, + string? LastAdvisoryId, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); + + public static CiscoCursor Empty { get; } = new(null, null, EmptyGuidCollection, EmptyGuidCollection); + + public BsonDocument ToBson() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + if (!string.IsNullOrWhiteSpace(LastAdvisoryId)) + { + document["lastAdvisoryId"] = LastAdvisoryId; + } + + return document; + } + + public static CiscoCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastModified = null; + if (document.TryGetValue("lastModified", out var lastModifiedValue)) + { + lastModified = lastModifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + string? lastAdvisoryId = null; + if (document.TryGetValue("lastAdvisoryId", out var idValue) && idValue.BsonType == BsonType.String) + { + var value = idValue.AsString.Trim(); + if (value.Length > 0) + { + lastAdvisoryId = value; + } + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new CiscoCursor(lastModified, lastAdvisoryId, pendingDocuments, pendingMappings); + } + + public CiscoCursor WithCheckpoint(DateTimeOffset lastModified, string advisoryId) + => this with + { + LastModified = lastModified.ToUniversalTime(), + LastAdvisoryId = string.IsNullOrWhiteSpace(advisoryId) ? null : advisoryId.Trim(), + }; + + public CiscoCursor WithPendingDocuments(IEnumerable? documents) + => this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidCollection }; + + public CiscoCursor WithPendingMappings(IEnumerable? mappings) + => this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidCollection }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string key) + { + if (!document.TryGetValue(key, out var value) || value is not BsonArray array) + { + return EmptyGuidCollection; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoDiagnostics.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoDiagnostics.cs new file mode 100644 index 00000000..8c421a98 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoDiagnostics.cs @@ -0,0 +1,82 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +public sealed class CiscoDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Cisco"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _mapSuccess; + private readonly Counter _mapFailures; + private readonly Histogram _mapAffected; + + public CiscoDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchDocuments = _meter.CreateCounter( + name: "cisco.fetch.documents", + unit: "documents", + description: "Number of Cisco advisories fetched."); + _fetchFailures = _meter.CreateCounter( + name: "cisco.fetch.failures", + unit: "operations", + description: "Number of Cisco fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "cisco.fetch.unchanged", + unit: "documents", + description: "Number of Cisco advisories skipped because they were unchanged."); + _parseSuccess = _meter.CreateCounter( + name: "cisco.parse.success", + unit: "documents", + description: "Number of Cisco documents parsed successfully."); + _parseFailures = _meter.CreateCounter( + name: "cisco.parse.failures", + unit: "documents", + description: "Number of Cisco documents that failed to parse."); + _mapSuccess = _meter.CreateCounter( + name: "cisco.map.success", + unit: "documents", + description: "Number of Cisco advisories mapped successfully."); + _mapFailures = _meter.CreateCounter( + name: "cisco.map.failures", + unit: "documents", + description: "Number of Cisco advisories that failed to map to canonical form."); + _mapAffected = _meter.CreateHistogram( + name: "cisco.map.affected.packages", + unit: "packages", + description: "Distribution of affected package counts emitted per Cisco advisory."); + } + + public Meter Meter => _meter; + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void MapSuccess() => _mapSuccess.Add(1); + + public void MapFailure() => _mapFailures.Add(1); + + public void MapAffected(int count) + { + if (count >= 0) + { + _mapAffected.Record(count); + } + } + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoDtoFactory.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoDtoFactory.cs new file mode 100644 index 00000000..e2d879b8 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoDtoFactory.cs @@ -0,0 +1,190 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +public class CiscoDtoFactory +{ + private readonly ICiscoCsafClient _csafClient; + private readonly ILogger _logger; + + public CiscoDtoFactory(ICiscoCsafClient csafClient, ILogger logger) + { + _csafClient = csafClient ?? throw new ArgumentNullException(nameof(csafClient)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CreateAsync(CiscoRawAdvisory raw, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(raw); + + var advisoryId = raw.AdvisoryId?.Trim(); + if (string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("Cisco advisory is missing advisoryId."); + } + + var title = string.IsNullOrWhiteSpace(raw.AdvisoryTitle) ? advisoryId : raw.AdvisoryTitle!.Trim(); + var severity = SeverityNormalization.Normalize(raw.Sir); + var published = ParseDate(raw.FirstPublished); + var updated = ParseDate(raw.LastUpdated); + + CiscoCsafData? csafData = null; + if (!string.IsNullOrWhiteSpace(raw.CsafUrl)) + { + var csafContent = await _csafClient.TryFetchAsync(raw.CsafUrl, cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(csafContent)) + { + try + { + csafData = CiscoCsafParser.Parse(csafContent!); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Cisco CSAF payload parsing failed for {AdvisoryId}", advisoryId); + } + } + } + + var products = BuildProducts(raw, csafData); + var cves = NormalizeList(raw.Cves); + var bugIds = NormalizeList(raw.BugIds); + var cvss = ParseDouble(raw.CvssBaseScore); + + return new CiscoAdvisoryDto( + AdvisoryId: advisoryId, + Title: title, + Summary: string.IsNullOrWhiteSpace(raw.Summary) ? null : raw.Summary!.Trim(), + Severity: severity, + Published: published, + Updated: updated, + PublicationUrl: NormalizeUrl(raw.PublicationUrl), + CsafUrl: NormalizeUrl(raw.CsafUrl), + CvrfUrl: NormalizeUrl(raw.CvrfUrl), + CvssBaseScore: cvss, + Cves: cves, + BugIds: bugIds, + Products: products); + } + + private static IReadOnlyList BuildProducts(CiscoRawAdvisory raw, CiscoCsafData? csafData) + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (csafData is not null) + { + foreach (var entry in csafData.ProductStatuses) + { + var productId = entry.Key; + var name = csafData.Products.TryGetValue(productId, out var product) + ? product.Name + : productId; + + var statuses = NormalizeStatuses(entry.Value); + map[name] = new CiscoAffectedProductDto( + Name: name, + ProductId: productId, + Version: raw.Version?.Trim(), + Statuses: statuses); + } + } + + var rawProducts = NormalizeList(raw.ProductNames); + foreach (var productName in rawProducts) + { + if (map.ContainsKey(productName)) + { + continue; + } + + map[productName] = new CiscoAffectedProductDto( + Name: productName, + ProductId: null, + Version: raw.Version?.Trim(), + Statuses: new[] { AffectedPackageStatusCatalog.KnownAffected }); + } + + return map.Count == 0 + ? Array.Empty() + : map.Values + .OrderBy(static p => p.Name, StringComparer.OrdinalIgnoreCase) + .ThenBy(static p => p.ProductId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyCollection NormalizeStatuses(IEnumerable statuses) + { + var set = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (var status in statuses) + { + if (AffectedPackageStatusCatalog.TryNormalize(status, out var normalized)) + { + set.Add(normalized); + } + else if (!string.IsNullOrWhiteSpace(status)) + { + set.Add(status.Trim().ToLowerInvariant()); + } + } + + if (set.Count == 0) + { + set.Add(AffectedPackageStatusCatalog.KnownAffected); + } + + return set; + } + + private static IReadOnlyList NormalizeList(IEnumerable? items) + { + if (items is null) + { + return Array.Empty(); + } + + var set = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (var item in items) + { + if (!string.IsNullOrWhiteSpace(item)) + { + set.Add(item.Trim()); + } + } + + return set.Count == 0 ? Array.Empty() : set.ToArray(); + } + + private static double? ParseDouble(string? value) + => double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) + ? parsed + : null; + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return parsed.ToUniversalTime(); + } + + return null; + } + + private static string? NormalizeUrl(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return Uri.TryCreate(value.Trim(), UriKind.Absolute, out var uri) ? uri.ToString() : null; + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoMapper.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoMapper.cs new file mode 100644 index 00000000..5477b59b --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoMapper.cs @@ -0,0 +1,263 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Packages; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +public static class CiscoMapper +{ + public static Advisory Map(CiscoAdvisoryDto dto, DocumentRecord document, DtoRecord dtoRecord) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + + var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); + var fetchProvenance = new AdvisoryProvenance( + VndrCiscoConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mapProvenance = new AdvisoryProvenance( + VndrCiscoConnectorPlugin.SourceName, + "map", + dto.AdvisoryId, + recordedAt); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var affected = BuildAffectedPackages(dto, recordedAt); + + return new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title, + summary: dto.Summary, + language: "en", + published: dto.Published, + modified: dto.Updated, + severity: dto.Severity, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affected, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mapProvenance }); + } + + private static IReadOnlyList BuildAliases(CiscoAdvisoryDto dto) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId, + }; + + foreach (var cve in dto.Cves) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + set.Add(cve.Trim()); + } + } + + foreach (var bugId in dto.BugIds) + { + if (!string.IsNullOrWhiteSpace(bugId)) + { + set.Add(bugId.Trim()); + } + } + + if (dto.PublicationUrl is not null) + { + set.Add(dto.PublicationUrl); + } + + return set.Count == 0 + ? Array.Empty() + : set.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static IReadOnlyList BuildReferences(CiscoAdvisoryDto dto, DateTimeOffset recordedAt) + { + var list = new List(3); + AddReference(list, dto.PublicationUrl, "publication", recordedAt); + AddReference(list, dto.CvrfUrl, "cvrf", recordedAt); + AddReference(list, dto.CsafUrl, "csaf", recordedAt); + + return list.Count == 0 + ? Array.Empty() + : list.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static void AddReference(ICollection references, string? url, string kind, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) + { + return; + } + + var provenance = new AdvisoryProvenance( + VndrCiscoConnectorPlugin.SourceName, + $"reference:{kind}", + uri.ToString(), + recordedAt); + + try + { + references.Add(new AdvisoryReference( + url: uri.ToString(), + kind: kind, + sourceTag: null, + summary: null, + provenance: provenance)); + } + catch (ArgumentException) + { + // ignore invalid URLs + } + } + + private static IReadOnlyList BuildAffectedPackages(CiscoAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.Products.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Products.Count); + foreach (var product in dto.Products) + { + if (string.IsNullOrWhiteSpace(product.Name)) + { + continue; + } + + var range = BuildVersionRange(product, recordedAt); + var statuses = BuildStatuses(product, recordedAt); + var provenance = new[] + { + new AdvisoryProvenance( + VndrCiscoConnectorPlugin.SourceName, + "affected", + product.ProductId ?? product.Name, + recordedAt), + }; + + packages.Add(new AffectedPackage( + type: AffectedPackageTypes.Vendor, + identifier: product.Name, + platform: null, + versionRanges: range is null ? Array.Empty() : new[] { range }, + statuses: statuses, + provenance: provenance, + normalizedVersions: Array.Empty())); + } + + return packages.Count == 0 + ? Array.Empty() + : packages.OrderBy(static p => p.Identifier, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static AffectedVersionRange? BuildVersionRange(CiscoAffectedProductDto product, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(product.Version)) + { + return null; + } + + var version = product.Version.Trim(); + RangePrimitives? primitives = null; + string rangeKind = "vendor"; + string? rangeExpression = version; + + if (PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized)) + { + var semver = new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: true, + ConstraintExpression: null, + ExactValue: normalized); + + primitives = new RangePrimitives(semver, null, null, BuildVendorExtensions(product)); + rangeKind = "semver"; + rangeExpression = normalized; + } + else + { + primitives = new RangePrimitives(null, null, null, BuildVendorExtensions(product, includeVersion: true)); + } + + var provenance = new AdvisoryProvenance( + VndrCiscoConnectorPlugin.SourceName, + "range", + product.ProductId ?? product.Name, + recordedAt); + + return new AffectedVersionRange( + rangeKind: rangeKind, + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: rangeExpression, + provenance: provenance, + primitives: primitives); + } + + private static IReadOnlyDictionary? BuildVendorExtensions(CiscoAffectedProductDto product, bool includeVersion = false) + { + var dictionary = new Dictionary(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(product.ProductId)) + { + dictionary["cisco.productId"] = product.ProductId!; + } + + if (includeVersion && !string.IsNullOrWhiteSpace(product.Version)) + { + dictionary["cisco.version.raw"] = product.Version!; + } + + return dictionary.Count == 0 ? null : dictionary; + } + + private static IReadOnlyList BuildStatuses(CiscoAffectedProductDto product, DateTimeOffset recordedAt) + { + if (product.Statuses is null || product.Statuses.Count == 0) + { + return Array.Empty(); + } + + var list = new List(product.Statuses.Count); + foreach (var status in product.Statuses) + { + if (!AffectedPackageStatusCatalog.TryNormalize(status, out var normalized) + || string.IsNullOrWhiteSpace(normalized)) + { + continue; + } + + var provenance = new AdvisoryProvenance( + VndrCiscoConnectorPlugin.SourceName, + "status", + product.ProductId ?? product.Name, + recordedAt); + + list.Add(new AffectedPackageStatus(normalized, provenance)); + } + + return list.Count == 0 ? Array.Empty() : list; + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs new file mode 100644 index 00000000..7fdecae1 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs @@ -0,0 +1,101 @@ +using System.IO; +using System.Net; +using System.Net.Http.Headers; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +internal sealed class CiscoOAuthMessageHandler : DelegatingHandler +{ + private readonly CiscoAccessTokenProvider _tokenProvider; + private readonly ILogger _logger; + + public CiscoOAuthMessageHandler( + CiscoAccessTokenProvider tokenProvider, + ILogger logger) + { + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + HttpRequestMessage? retryTemplate = null; + try + { + retryTemplate = await CloneRequestAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (IOException) + { + // Unable to buffer content; retry will fail if needed. + retryTemplate = null; + } + + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false)); + var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.Unauthorized) + { + return response; + } + + response.Dispose(); + _logger.LogWarning("Cisco openVuln request returned 401 Unauthorized; refreshing access token."); + await _tokenProvider.RefreshAsync(cancellationToken).ConfigureAwait(false); + + if (retryTemplate is null) + { + _tokenProvider.Invalidate(); + throw new HttpRequestException("Cisco openVuln request returned 401 Unauthorized and could not be retried."); + } + + retryTemplate.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false)); + + try + { + var retryResponse = await base.SendAsync(retryTemplate, cancellationToken).ConfigureAwait(false); + if (retryResponse.StatusCode == HttpStatusCode.Unauthorized) + { + _tokenProvider.Invalidate(); + } + + return retryResponse; + } + finally + { + retryTemplate.Dispose(); + } + } + + private static async Task CloneRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var clone = new HttpRequestMessage(request.Method, request.RequestUri) + { + Version = request.Version, + VersionPolicy = request.VersionPolicy, + }; + + foreach (var header in request.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (request.Content is not null) + { + using var memory = new MemoryStream(); + await request.Content.CopyToAsync(memory, cancellationToken).ConfigureAwait(false); + memory.Position = 0; + var buffer = memory.ToArray(); + var contentClone = new ByteArrayContent(buffer); + foreach (var header in request.Content.Headers) + { + contentClone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + clone.Content = contentClone; + } + + return clone; + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOpenVulnClient.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOpenVulnClient.cs new file mode 100644 index 00000000..38bdb5ac --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoOpenVulnClient.cs @@ -0,0 +1,196 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +public sealed class CiscoOpenVulnClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + }; + + private readonly SourceFetchService _fetchService; + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + private readonly string _sourceName; + + public CiscoOpenVulnClient( + SourceFetchService fetchService, + IOptionsMonitor options, + ILogger logger, + string sourceName) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _sourceName = sourceName ?? throw new ArgumentNullException(nameof(sourceName)); + } + + internal async Task FetchAsync(DateOnly date, int pageIndex, CancellationToken cancellationToken) + { + var options = _options.CurrentValue; + var requestUri = options.BuildLastModifiedUri(date, pageIndex, options.PageSize); + var request = new SourceFetchRequest(CiscoOptions.HttpClientName, _sourceName, requestUri) + { + AcceptHeaders = new[] { "application/json" }, + TimeoutOverride = options.RequestTimeout, + }; + + var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Content is null) + { + _logger.LogDebug("Cisco openVuln request returned empty payload for {Uri} (status {Status})", requestUri, result.StatusCode); + return null; + } + + return CiscoAdvisoryPage.Parse(result.Content); + } +} + +internal sealed record CiscoAdvisoryPage( + IReadOnlyList Advisories, + CiscoPagination Pagination) +{ + public bool HasMore => Pagination.PageIndex < Pagination.TotalPages; + + public static CiscoAdvisoryPage Parse(byte[] content) + { + using var document = JsonDocument.Parse(content); + var root = document.RootElement; + var advisories = new List(); + + if (root.TryGetProperty("advisories", out var advisoriesElement) && advisoriesElement.ValueKind == JsonValueKind.Array) + { + foreach (var advisory in advisoriesElement.EnumerateArray()) + { + if (!TryCreateItem(advisory, out var item)) + { + continue; + } + + advisories.Add(item); + } + } + + var pagination = CiscoPagination.FromJson(root.TryGetProperty("pagination", out var paginationElement) ? paginationElement : default); + return new CiscoAdvisoryPage(advisories, pagination); + } + + private static bool TryCreateItem(JsonElement advisory, [NotNullWhen(true)] out CiscoAdvisoryItem? item) + { + var rawJson = advisory.GetRawText(); + var advisoryId = GetString(advisory, "advisoryId"); + if (string.IsNullOrWhiteSpace(advisoryId)) + { + item = null; + return false; + } + + var lastUpdated = ParseDate(GetString(advisory, "lastUpdated")); + var firstPublished = ParseDate(GetString(advisory, "firstPublished")); + var severity = GetString(advisory, "sir"); + var publicationUrl = GetString(advisory, "publicationUrl"); + var csafUrl = GetString(advisory, "csafUrl"); + var cvrfUrl = GetString(advisory, "cvrfUrl"); + var cvss = GetString(advisory, "cvssBaseScore"); + + var cves = ReadStringArray(advisory, "cves"); + var bugIds = ReadStringArray(advisory, "bugIDs"); + var productNames = ReadStringArray(advisory, "productNames"); + + item = new CiscoAdvisoryItem( + advisoryId, + lastUpdated, + firstPublished, + severity, + publicationUrl, + csafUrl, + cvrfUrl, + cvss, + cves, + bugIds, + productNames, + rawJson); + return true; + } + + private static string? GetString(JsonElement element, string propertyName) + => element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, out var parsed)) + { + return parsed.ToUniversalTime(); + } + + return null; + } + + private static IReadOnlyList ReadStringArray(JsonElement element, string property) + { + if (!element.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var results = new List(); + foreach (var child in value.EnumerateArray()) + { + if (child.ValueKind == JsonValueKind.String) + { + var text = child.GetString(); + if (!string.IsNullOrWhiteSpace(text)) + { + results.Add(text.Trim()); + } + } + } + + return results; + } +} + +internal sealed record CiscoAdvisoryItem( + string AdvisoryId, + DateTimeOffset? LastUpdated, + DateTimeOffset? FirstPublished, + string? Severity, + string? PublicationUrl, + string? CsafUrl, + string? CvrfUrl, + string? CvssBaseScore, + IReadOnlyList Cves, + IReadOnlyList BugIds, + IReadOnlyList ProductNames, + string RawJson) +{ + public byte[] GetRawBytes() => Encoding.UTF8.GetBytes(RawJson); +} + +internal sealed record CiscoPagination(int PageIndex, int PageSize, int TotalPages, int TotalRecords) +{ + public static CiscoPagination FromJson(JsonElement element) + { + var pageIndex = element.TryGetProperty("pageIndex", out var index) && index.TryGetInt32(out var parsedIndex) ? parsedIndex : 1; + var pageSize = element.TryGetProperty("pageSize", out var size) && size.TryGetInt32(out var parsedSize) ? parsedSize : 0; + var totalPages = element.TryGetProperty("totalPages", out var pages) && pages.TryGetInt32(out var parsedPages) ? parsedPages : pageIndex; + var totalRecords = element.TryGetProperty("totalRecords", out var records) && records.TryGetInt32(out var parsedRecords) ? parsedRecords : 0; + return new CiscoPagination(pageIndex, pageSize, totalPages, totalRecords); + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoRawAdvisory.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoRawAdvisory.cs new file mode 100644 index 00000000..4c0dffc5 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Internal/CiscoRawAdvisory.cs @@ -0,0 +1,64 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal; + +public class CiscoRawAdvisory +{ + [JsonPropertyName("advisoryId")] + public string? AdvisoryId { get; set; } + + [JsonPropertyName("advisoryTitle")] + public string? AdvisoryTitle { get; set; } + + [JsonPropertyName("publicationUrl")] + public string? PublicationUrl { get; set; } + + [JsonPropertyName("cvrfUrl")] + public string? CvrfUrl { get; set; } + + [JsonPropertyName("csafUrl")] + public string? CsafUrl { get; set; } + + [JsonPropertyName("summary")] + public string? Summary { get; set; } + + [JsonPropertyName("sir")] + public string? Sir { get; set; } + + [JsonPropertyName("firstPublished")] + public string? FirstPublished { get; set; } + + [JsonPropertyName("lastUpdated")] + public string? LastUpdated { get; set; } + + [JsonPropertyName("productNames")] + public List? ProductNames { get; set; } + + [JsonPropertyName("version")] + public string? Version { get; set; } + + [JsonPropertyName("iosRelease")] + public string? IosRelease { get; set; } + + [JsonPropertyName("cves")] + public List? Cves { get; set; } + + [JsonPropertyName("bugIDs")] + public List? BugIds { get; set; } + + [JsonPropertyName("cvssBaseScore")] + public string? CvssBaseScore { get; set; } + + [JsonPropertyName("cvssTemporalScore")] + public string? CvssTemporalScore { get; set; } + + [JsonPropertyName("cvssEnvironmentalScore")] + public string? CvssEnvironmentalScore { get; set; } + + [JsonPropertyName("cvssBaseScoreVersion2")] + public string? CvssBaseScoreV2 { get; set; } + + [JsonPropertyName("status")] + public string? Status { get; set; } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Configuration/MsrcOptions.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Configuration/MsrcOptions.cs new file mode 100644 index 00000000..2bd43c38 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Configuration/MsrcOptions.cs @@ -0,0 +1,132 @@ +using System.Globalization; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; + +public sealed class MsrcOptions +{ + public const string HttpClientName = "concelier.source.vndr.msrc"; + public const string TokenClientName = "concelier.source.vndr.msrc.token"; + + public Uri BaseUri { get; set; } = new("https://api.msrc.microsoft.com/sug/v2.0/", UriKind.Absolute); + + public string Locale { get; set; } = "en-US"; + + public string ApiVersion { get; set; } = "2024-08-01"; + + /// + /// Azure AD tenant identifier used for client credential flow. + /// + public string TenantId { get; set; } = string.Empty; + + /// + /// Azure AD application (client) identifier. + /// + public string ClientId { get; set; } = string.Empty; + + /// + /// Azure AD client secret used for token acquisition. + /// + public string ClientSecret { get; set; } = string.Empty; + + /// + /// Scope requested during client-credential token acquisition. + /// + public string Scope { get; set; } = "api://api.msrc.microsoft.com/.default"; + + /// + /// Maximum advisories to fetch per cycle. + /// + public int MaxAdvisoriesPerFetch { get; set; } = 200; + + /// + /// Page size used when iterating the MSRC API. + /// + public int PageSize { get; set; } = 100; + + /// + /// Overlap window added when resuming from the last modified cursor. + /// + public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10); + + /// + /// When enabled the connector downloads the CVRF artefact referenced by each advisory. + /// + public bool DownloadCvrf { get; set; } = false; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Optional lower bound for the initial sync if the cursor is empty. + /// + public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30); + + public void Validate() + { + if (BaseUri is null || !BaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("MSRC base URI must be absolute."); + } + + if (string.IsNullOrWhiteSpace(Locale)) + { + throw new InvalidOperationException("Locale must be provided."); + } + + if (!string.IsNullOrWhiteSpace(Locale) && !CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidOperationException($"Locale '{Locale}' is not recognised."); + } + + if (string.IsNullOrWhiteSpace(ApiVersion)) + { + throw new InvalidOperationException("API version must be provided."); + } + + if (!Guid.TryParse(TenantId, out _)) + { + throw new InvalidOperationException("TenantId must be a valid GUID."); + } + + if (string.IsNullOrWhiteSpace(ClientId)) + { + throw new InvalidOperationException("ClientId must be provided."); + } + + if (string.IsNullOrWhiteSpace(ClientSecret)) + { + throw new InvalidOperationException("ClientSecret must be provided."); + } + + if (string.IsNullOrWhiteSpace(Scope)) + { + throw new InvalidOperationException("Scope must be provided."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero."); + } + + if (PageSize <= 0 || PageSize > 500) + { + throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500."); + } + + if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6)) + { + throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive."); + } + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcAdvisoryDto.cs new file mode 100644 index 00000000..34315b47 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcAdvisoryDto.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +public sealed record MsrcAdvisoryDto +{ + public string AdvisoryId { get; init; } = string.Empty; + + public string Title { get; init; } = string.Empty; + + public string? Description { get; init; } + + public string? Severity { get; init; } + + public DateTimeOffset? ReleaseDate { get; init; } + + public DateTimeOffset? LastModifiedDate { get; init; } + + public IReadOnlyList CveIds { get; init; } = Array.Empty(); + + public IReadOnlyList KbIds { get; init; } = Array.Empty(); + + public IReadOnlyList Threats { get; init; } = Array.Empty(); + + public IReadOnlyList Remediations { get; init; } = Array.Empty(); + + public IReadOnlyList Products { get; init; } = Array.Empty(); + + public double? CvssBaseScore { get; init; } + + public string? CvssVector { get; init; } + + public string? ReleaseNoteUrl { get; init; } + + public string? CvrfUrl { get; init; } +} + +public sealed record MsrcAdvisoryThreat(string Type, string? Description, string? Severity); + +public sealed record MsrcAdvisoryRemediation(string Type, string? Description, string? Url, string? Kb); + +public sealed record MsrcAdvisoryProduct( + string Identifier, + string? ProductName, + string? Platform, + string? Architecture, + string? BuildNumber, + string? Cpe); diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcApiClient.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcApiClient.cs new file mode 100644 index 00000000..de1bc678 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcApiClient.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +public sealed class MsrcApiClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + WriteIndented = false, + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMsrcTokenProvider _tokenProvider; + private readonly MsrcOptions _options; + private readonly ILogger _logger; + + public MsrcApiClient( + IHttpClientFactory httpClientFactory, + IMsrcTokenProvider tokenProvider, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> FetchSummariesAsync(DateTimeOffset fromInclusive, DateTimeOffset toExclusive, CancellationToken cancellationToken) + { + var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false); + + var results = new List(); + var requestUri = BuildSummaryUri(fromInclusive, toExclusive); + + while (requestUri is not null) + { + using var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new HttpRequestException($"MSRC summary fetch failed with {(int)response.StatusCode}. Body: {preview}"); + } + + var payload = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false) + ?? new MsrcSummaryResponse(); + + results.AddRange(payload.Value); + + if (string.IsNullOrWhiteSpace(payload.NextLink)) + { + break; + } + + requestUri = new Uri(payload.NextLink, UriKind.Absolute); + } + + return results; + } + + public Uri BuildDetailUri(string vulnerabilityId) + { + var uri = CreateDetailUriInternal(vulnerabilityId); + return uri; + } + + public async Task FetchDetailAsync(string vulnerabilityId, CancellationToken cancellationToken) + { + var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false); + var uri = CreateDetailUriInternal(vulnerabilityId); + + using var request = new HttpRequestMessage(HttpMethod.Get, uri); + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new HttpRequestException($"MSRC detail fetch failed for {vulnerabilityId} with {(int)response.StatusCode}. Body: {preview}"); + } + + return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task CreateAuthenticatedClientAsync(CancellationToken cancellationToken) + { + var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false); + var client = _httpClientFactory.CreateClient(MsrcOptions.HttpClientName); + client.DefaultRequestHeaders.Remove("Authorization"); + client.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}"); + client.DefaultRequestHeaders.Remove("Accept"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + client.DefaultRequestHeaders.Remove("api-version"); + client.DefaultRequestHeaders.Add("api-version", _options.ApiVersion); + client.DefaultRequestHeaders.Remove("Accept-Language"); + client.DefaultRequestHeaders.Add("Accept-Language", _options.Locale); + return client; + } + + private Uri BuildSummaryUri(DateTimeOffset fromInclusive, DateTimeOffset toExclusive) + { + var builder = new StringBuilder(); + builder.Append(_options.BaseUri.ToString().TrimEnd('/')); + builder.Append("/vulnerabilities?"); + builder.Append("$top=").Append(_options.PageSize); + builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(fromInclusive.ToUniversalTime().ToString("O"))); + builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(toExclusive.ToUniversalTime().ToString("O"))); + builder.Append("&$orderby=lastModifiedDate"); + builder.Append("&locale=").Append(Uri.EscapeDataString(_options.Locale)); + builder.Append("&api-version=").Append(Uri.EscapeDataString(_options.ApiVersion)); + + return new Uri(builder.ToString(), UriKind.Absolute); + } + + private Uri CreateDetailUriInternal(string vulnerabilityId) + { + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + throw new ArgumentException("Vulnerability identifier must be provided.", nameof(vulnerabilityId)); + } + + var baseUri = _options.BaseUri.ToString().TrimEnd('/'); + var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(_options.ApiVersion)}&locale={Uri.EscapeDataString(_options.Locale)}"; + return new Uri(path, UriKind.Absolute); + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcCursor.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcCursor.cs new file mode 100644 index 00000000..7166dd40 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcCursor.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +internal sealed record MsrcCursor( + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + DateTimeOffset? LastModifiedCursor) +{ + private static readonly IReadOnlyCollection EmptyGuidSet = Array.Empty(); + + public static MsrcCursor Empty { get; } = new(EmptyGuidSet, EmptyGuidSet, null); + + public MsrcCursor WithPendingDocuments(IEnumerable documents) + => this with { PendingDocuments = Distinct(documents) }; + + public MsrcCursor WithPendingMappings(IEnumerable mappings) + => this with { PendingMappings = Distinct(mappings) }; + + public MsrcCursor WithLastModifiedCursor(DateTimeOffset? timestamp) + => this with { LastModifiedCursor = timestamp }; + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastModifiedCursor.HasValue) + { + document["lastModifiedCursor"] = LastModifiedCursor.Value.UtcDateTime; + } + + return document; + } + + public static MsrcCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var lastModified = document.TryGetValue("lastModifiedCursor", out var value) + ? ParseDate(value) + : null; + + return new MsrcCursor(pendingDocuments, pendingMappings, lastModified); + } + + private static IReadOnlyCollection Distinct(IEnumerable? values) + => values?.Distinct().ToArray() ?? EmptyGuidSet; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidSet; + } + + var items = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element?.ToString(), out var id)) + { + items.Add(id); + } + } + + return items; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDetailDto.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDetailDto.cs new file mode 100644 index 00000000..8e108b8d --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDetailDto.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +public sealed record MsrcVulnerabilityDetailDto +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("vulnerabilityId")] + public string VulnerabilityId { get; init; } = string.Empty; + + [JsonPropertyName("cveNumber")] + public string? CveNumber { get; init; } + + [JsonPropertyName("cveNumbers")] + public IReadOnlyList CveNumbers { get; init; } = Array.Empty(); + + [JsonPropertyName("title")] + public string Title { get; init; } = string.Empty; + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("releaseDate")] + public DateTimeOffset? ReleaseDate { get; init; } + + [JsonPropertyName("lastModifiedDate")] + public DateTimeOffset? LastModifiedDate { get; init; } + + [JsonPropertyName("severity")] + public string? Severity { get; init; } + + [JsonPropertyName("threats")] + public IReadOnlyList Threats { get; init; } = Array.Empty(); + + [JsonPropertyName("remediations")] + public IReadOnlyList Remediations { get; init; } = Array.Empty(); + + [JsonPropertyName("affectedProducts")] + public IReadOnlyList AffectedProducts { get; init; } = Array.Empty(); + + [JsonPropertyName("cvssV3")] + public MsrcCvssDto? Cvss { get; init; } + + [JsonPropertyName("releaseNoteUrl")] + public string? ReleaseNoteUrl { get; init; } + + [JsonPropertyName("cvrfUrl")] + public string? CvrfUrl { get; init; } +} + +public sealed record MsrcThreatDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("severity")] + public string? Severity { get; init; } +} + +public sealed record MsrcRemediationDto +{ + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("url")] + public string? Url { get; init; } + + [JsonPropertyName("kbNumber")] + public string? KbNumber { get; init; } +} + +public sealed record MsrcAffectedProductDto +{ + [JsonPropertyName("productId")] + public string? ProductId { get; init; } + + [JsonPropertyName("productName")] + public string? ProductName { get; init; } + + [JsonPropertyName("cpe")] + public string? Cpe { get; init; } + + [JsonPropertyName("platform")] + public string? Platform { get; init; } + + [JsonPropertyName("architecture")] + public string? Architecture { get; init; } + + [JsonPropertyName("buildNumber")] + public string? BuildNumber { get; init; } +} + +public sealed record MsrcCvssDto +{ + [JsonPropertyName("baseScore")] + public double? BaseScore { get; init; } + + [JsonPropertyName("vectorString")] + public string? VectorString { get; init; } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDetailParser.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDetailParser.cs new file mode 100644 index 00000000..b9c57ea7 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDetailParser.cs @@ -0,0 +1,71 @@ +using System; +using System.Linq; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +public sealed class MsrcDetailParser +{ + public MsrcAdvisoryDto Parse(MsrcVulnerabilityDetailDto detail) + { + ArgumentNullException.ThrowIfNull(detail); + + var advisoryId = string.IsNullOrWhiteSpace(detail.VulnerabilityId) ? detail.Id : detail.VulnerabilityId; + var cveIds = detail.CveNumbers?.Where(static c => !string.IsNullOrWhiteSpace(c)).Select(static c => c.Trim()).ToArray() + ?? (string.IsNullOrWhiteSpace(detail.CveNumber) ? Array.Empty() : new[] { detail.CveNumber! }); + + var kbIds = detail.Remediations? + .Where(static remediation => !string.IsNullOrWhiteSpace(remediation.KbNumber)) + .Select(static remediation => remediation.KbNumber!.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? Array.Empty(); + + return new MsrcAdvisoryDto + { + AdvisoryId = advisoryId, + Title = string.IsNullOrWhiteSpace(detail.Title) ? advisoryId : detail.Title.Trim(), + Description = detail.Description, + Severity = detail.Severity, + ReleaseDate = detail.ReleaseDate, + LastModifiedDate = detail.LastModifiedDate, + CveIds = cveIds, + KbIds = kbIds, + Threats = detail.Threats?.Select(static threat => new MsrcAdvisoryThreat( + threat.Type ?? "unspecified", + threat.Description, + threat.Severity)).ToArray() ?? Array.Empty(), + Remediations = detail.Remediations?.Select(static remediation => new MsrcAdvisoryRemediation( + remediation.Type ?? "unspecified", + remediation.Description, + remediation.Url, + remediation.KbNumber)).ToArray() ?? Array.Empty(), + Products = detail.AffectedProducts?.Select(product => + new MsrcAdvisoryProduct( + BuildProductIdentifier(product), + product.ProductName, + product.Platform, + product.Architecture, + product.BuildNumber, + product.Cpe)).ToArray() ?? Array.Empty(), + CvssBaseScore = detail.Cvss?.BaseScore, + CvssVector = detail.Cvss?.VectorString, + ReleaseNoteUrl = detail.ReleaseNoteUrl, + CvrfUrl = detail.CvrfUrl, + }; + } + + private static string BuildProductIdentifier(MsrcAffectedProductDto product) + { + var name = string.IsNullOrWhiteSpace(product.ProductName) ? product.ProductId : product.ProductName; + if (string.IsNullOrWhiteSpace(name)) + { + name = "Unknown Product"; + } + + if (!string.IsNullOrWhiteSpace(product.BuildNumber)) + { + return $"{name} build {product.BuildNumber}"; + } + + return name; + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDiagnostics.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDiagnostics.cs new file mode 100644 index 00000000..663548ef --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDiagnostics.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +public sealed class MsrcDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Concelier.Connector.Vndr.Msrc"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _summaryFetchAttempts; + private readonly Counter _summaryFetchSuccess; + private readonly Counter _summaryFetchFailures; + private readonly Histogram _summaryItemCount; + private readonly Histogram _summaryWindowHours; + private readonly Counter _detailFetchAttempts; + private readonly Counter _detailFetchSuccess; + private readonly Counter _detailFetchNotModified; + private readonly Counter _detailFetchFailures; + private readonly Histogram _detailEnqueued; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Histogram _parseProductCount; + private readonly Histogram _parseKbCount; + private readonly Counter _mapSuccess; + private readonly Counter _mapFailures; + private readonly Histogram _mapAliasCount; + private readonly Histogram _mapAffectedCount; + + public MsrcDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _summaryFetchAttempts = _meter.CreateCounter("msrc.summary.fetch.attempts", "operations"); + _summaryFetchSuccess = _meter.CreateCounter("msrc.summary.fetch.success", "operations"); + _summaryFetchFailures = _meter.CreateCounter("msrc.summary.fetch.failures", "operations"); + _summaryItemCount = _meter.CreateHistogram("msrc.summary.items.count", "items"); + _summaryWindowHours = _meter.CreateHistogram("msrc.summary.window.hours", "hours"); + _detailFetchAttempts = _meter.CreateCounter("msrc.detail.fetch.attempts", "operations"); + _detailFetchSuccess = _meter.CreateCounter("msrc.detail.fetch.success", "operations"); + _detailFetchNotModified = _meter.CreateCounter("msrc.detail.fetch.not_modified", "operations"); + _detailFetchFailures = _meter.CreateCounter("msrc.detail.fetch.failures", "operations"); + _detailEnqueued = _meter.CreateHistogram("msrc.detail.enqueued.count", "documents"); + _parseSuccess = _meter.CreateCounter("msrc.parse.success", "documents"); + _parseFailures = _meter.CreateCounter("msrc.parse.failures", "documents"); + _parseProductCount = _meter.CreateHistogram("msrc.parse.products.count", "products"); + _parseKbCount = _meter.CreateHistogram("msrc.parse.kb.count", "kb"); + _mapSuccess = _meter.CreateCounter("msrc.map.success", "advisories"); + _mapFailures = _meter.CreateCounter("msrc.map.failures", "advisories"); + _mapAliasCount = _meter.CreateHistogram("msrc.map.aliases.count", "aliases"); + _mapAffectedCount = _meter.CreateHistogram("msrc.map.affected.count", "packages"); + } + + public void SummaryFetchAttempt() => _summaryFetchAttempts.Add(1); + + public void SummaryFetchSuccess(int count, double? windowHours) + { + _summaryFetchSuccess.Add(1); + if (count >= 0) + { + _summaryItemCount.Record(count); + } + + if (windowHours is { } value && value >= 0) + { + _summaryWindowHours.Record(value); + } + } + + public void SummaryFetchFailure(string reason) + => _summaryFetchFailures.Add(1, ReasonTag(reason)); + + public void DetailFetchAttempt() => _detailFetchAttempts.Add(1); + + public void DetailFetchSuccess() => _detailFetchSuccess.Add(1); + + public void DetailFetchNotModified() => _detailFetchNotModified.Add(1); + + public void DetailFetchFailure(string reason) + => _detailFetchFailures.Add(1, ReasonTag(reason)); + + public void DetailEnqueued(int count) + { + if (count >= 0) + { + _detailEnqueued.Record(count); + } + } + + public void ParseSuccess(int productCount, int kbCount) + { + _parseSuccess.Add(1); + if (productCount >= 0) + { + _parseProductCount.Record(productCount); + } + + if (kbCount >= 0) + { + _parseKbCount.Record(kbCount); + } + } + + public void ParseFailure(string reason) + => _parseFailures.Add(1, ReasonTag(reason)); + + public void MapSuccess(int aliasCount, int packageCount) + { + _mapSuccess.Add(1); + if (aliasCount >= 0) + { + _mapAliasCount.Record(aliasCount); + } + + if (packageCount >= 0) + { + _mapAffectedCount.Record(packageCount); + } + } + + public void MapFailure(string reason) + => _mapFailures.Add(1, ReasonTag(reason)); + + private static KeyValuePair ReasonTag(string reason) + => new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant()); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDocumentMetadata.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDocumentMetadata.cs new file mode 100644 index 00000000..87627e61 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcDocumentMetadata.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +internal static class MsrcDocumentMetadata +{ + public static Dictionary CreateMetadata(MsrcVulnerabilitySummary summary) + { + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["msrc.vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id, + ["msrc.id"] = summary.Id, + }; + + if (summary.LastModifiedDate.HasValue) + { + metadata["msrc.lastModified"] = summary.LastModifiedDate.Value.ToString("O"); + } + + if (summary.ReleaseDate.HasValue) + { + metadata["msrc.releaseDate"] = summary.ReleaseDate.Value.ToString("O"); + } + + if (!string.IsNullOrWhiteSpace(summary.CvrfUrl)) + { + metadata["msrc.cvrfUrl"] = summary.CvrfUrl!; + } + + if (summary.CveNumbers.Count > 0) + { + metadata["msrc.cves"] = string.Join(",", summary.CveNumbers); + } + + return metadata; + } + + public static Dictionary CreateCvrfMetadata(MsrcVulnerabilitySummary summary) + { + var metadata = CreateMetadata(summary); + metadata["msrc.cvrf"] = "true"; + return metadata; + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcMapper.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcMapper.cs new file mode 100644 index 00000000..265b8ed5 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcMapper.cs @@ -0,0 +1,239 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +internal static class MsrcMapper +{ + public static Advisory Map(MsrcAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var advisoryKey = dto.AdvisoryId; + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var affectedPackages = BuildPackages(dto, recordedAt); + var cvssMetrics = BuildCvss(dto, recordedAt); + + var provenance = new AdvisoryProvenance( + source: MsrcConnectorPlugin.SourceName, + kind: "advisory", + value: advisoryKey, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + + return new Advisory( + advisoryKey: advisoryKey, + title: dto.Title, + summary: dto.Description, + language: "en", + published: dto.ReleaseDate, + modified: dto.LastModifiedDate, + severity: NormalizeSeverity(dto.Severity), + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: cvssMetrics, + provenance: new[] { provenance }); + } + + private static IReadOnlyList BuildAliases(MsrcAdvisoryDto dto) + { + var aliases = new List { dto.AdvisoryId }; + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve); + } + } + + foreach (var kb in dto.KbIds) + { + if (!string.IsNullOrWhiteSpace(kb)) + { + aliases.Add(kb.StartsWith("KB", StringComparison.OrdinalIgnoreCase) ? kb : $"KB{kb}"); + } + } + + return aliases + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildReferences(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) + { + var references = new List(); + + if (!string.IsNullOrWhiteSpace(dto.ReleaseNoteUrl)) + { + references.Add(CreateReference(dto.ReleaseNoteUrl!, "details", recordedAt)); + } + + if (!string.IsNullOrWhiteSpace(dto.CvrfUrl)) + { + references.Add(CreateReference(dto.CvrfUrl!, "cvrf", recordedAt)); + } + + foreach (var remediation in dto.Remediations) + { + if (!string.IsNullOrWhiteSpace(remediation.Url)) + { + references.Add(CreateReference( + remediation.Url!, + string.Equals(remediation.Type, "security update", StringComparison.OrdinalIgnoreCase) ? "remediation" : remediation.Type ?? "reference", + recordedAt, + remediation.Description)); + } + } + + return references + .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static AdvisoryReference CreateReference(string url, string kind, DateTimeOffset recordedAt, string? summary = null) + => new( + url, + kind: kind.ToLowerInvariant(), + sourceTag: "msrc", + summary: summary, + provenance: new AdvisoryProvenance( + MsrcConnectorPlugin.SourceName, + "reference", + url, + recordedAt, + new[] { ProvenanceFieldMasks.References })); + + private static IReadOnlyList BuildPackages(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.Products.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Products.Count); + foreach (var product in dto.Products) + { + var identifier = string.IsNullOrWhiteSpace(product.Identifier) ? "Unknown Product" : product.Identifier; + var provenance = new AdvisoryProvenance( + MsrcConnectorPlugin.SourceName, + "package", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }); + + var notes = new List(); + if (!string.IsNullOrWhiteSpace(product.Platform)) + { + notes.Add($"platform:{product.Platform}"); + } + + if (!string.IsNullOrWhiteSpace(product.Architecture)) + { + notes.Add($"arch:{product.Architecture}"); + } + + if (!string.IsNullOrWhiteSpace(product.Cpe)) + { + notes.Add($"cpe:{product.Cpe}"); + } + + var range = !string.IsNullOrWhiteSpace(product.BuildNumber) + ? new[] + { + new AffectedVersionRange( + rangeKind: "custom", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: $"build:{product.BuildNumber}", + provenance: new AdvisoryProvenance( + MsrcConnectorPlugin.SourceName, + "package-range", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.VersionRanges })), + } + : Array.Empty(); + + var normalizedRules = !string.IsNullOrWhiteSpace(product.BuildNumber) + ? new[] + { + new NormalizedVersionRule( + scheme: "msrc.build", + type: NormalizedVersionRuleTypes.Exact, + value: product.BuildNumber, + notes: string.Join(";", notes.Where(static n => !string.IsNullOrWhiteSpace(n)))) + } + : Array.Empty(); + + packages.Add(new AffectedPackage( + type: AffectedPackageTypes.Vendor, + identifier: identifier, + platform: product.Platform, + versionRanges: range, + statuses: Array.Empty(), + provenance: new[] { provenance }, + normalizedVersions: normalizedRules)); + } + + return packages + .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildCvss(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.CvssBaseScore is null || string.IsNullOrWhiteSpace(dto.CvssVector)) + { + return Array.Empty(); + } + + var severity = CvssSeverityFromScore(dto.CvssBaseScore.Value); + + return new[] + { + new CvssMetric( + version: "3.1", + vector: dto.CvssVector!, + baseScore: dto.CvssBaseScore.Value, + baseSeverity: severity, + provenance: new AdvisoryProvenance( + MsrcConnectorPlugin.SourceName, + "cvss", + dto.AdvisoryId, + recordedAt, + new[] { ProvenanceFieldMasks.CvssMetrics })), + }; + } + + private static string CvssSeverityFromScore(double score) + => score switch + { + < 0 => "none", + < 4 => "low", + < 7 => "medium", + < 9 => "high", + _ => "critical", + }; + + private static string? NormalizeSeverity(string? severity) + { + if (string.IsNullOrWhiteSpace(severity)) + { + return null; + } + + return severity.Trim().ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcSummaryResponse.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcSummaryResponse.cs new file mode 100644 index 00000000..31fef565 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcSummaryResponse.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +public sealed record MsrcSummaryResponse +{ + [JsonPropertyName("value")] + public List Value { get; init; } = new(); + + [JsonPropertyName("@odata.nextLink")] + public string? NextLink { get; init; } +} + +public sealed record MsrcVulnerabilitySummary +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("vulnerabilityId")] + public string? VulnerabilityId { get; init; } + + [JsonPropertyName("cveNumber")] + public string? CveNumber { get; init; } + + [JsonPropertyName("cveNumbers")] + public IReadOnlyList CveNumbers { get; init; } = Array.Empty(); + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("releaseDate")] + public DateTimeOffset? ReleaseDate { get; init; } + + [JsonPropertyName("lastModifiedDate")] + public DateTimeOffset? LastModifiedDate { get; init; } + + [JsonPropertyName("severity")] + public string? Severity { get; init; } + + [JsonPropertyName("cvrfUrl")] + public string? CvrfUrl { get; init; } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcTokenProvider.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcTokenProvider.cs new file mode 100644 index 00000000..69291148 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Internal/MsrcTokenProvider.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal; + +public interface IMsrcTokenProvider +{ + Task GetAccessTokenAsync(CancellationToken cancellationToken); +} + +public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly MsrcOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly SemaphoreSlim _refreshLock = new(1, 1); + + private AccessToken? _currentToken; + + public MsrcTokenProvider( + IHttpClientFactory httpClientFactory, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task GetAccessTokenAsync(CancellationToken cancellationToken) + { + var token = _currentToken; + if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow())) + { + return token.Token; + } + + await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + token = _currentToken; + if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow())) + { + return token.Token; + } + + _logger.LogInformation("Requesting new MSRC access token"); + var client = _httpClientFactory.CreateClient(MsrcOptions.TokenClientName); + var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) + { + Content = new FormUrlEncodedContent(new Dictionary + { + ["client_id"] = _options.ClientId, + ["client_secret"] = _options.ClientSecret, + ["grant_type"] = "client_credentials", + ["scope"] = _options.Scope, + }), + }; + + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("AAD token response was null."); + + var expiresAt = _timeProvider.GetUtcNow().AddSeconds(payload.ExpiresIn - 60); + _currentToken = new AccessToken(payload.AccessToken, expiresAt); + return payload.AccessToken; + } + finally + { + _refreshLock.Release(); + } + } + + private Uri BuildTokenUri() + => new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token"); + + public void Dispose() => _refreshLock.Dispose(); + + private sealed record AccessToken(string Token, DateTimeOffset ExpiresAt) + { + public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt; + } + + private sealed record TokenResponse + { + [JsonPropertyName("access_token")] + public string AccessToken { get; init; } = string.Empty; + + [JsonPropertyName("expires_in")] + public int ExpiresIn { get; init; } + } +} diff --git a/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs index 9a0561c8..f6b529e9 100644 --- a/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Text; -using FluentAssertions; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -11,12 +11,13 @@ using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.Cisco.CSAF; using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Storage.Mongo; -using System.Collections.Immutable; -using System.IO.Abstractions.TestingHelpers; -using Xunit; -using System.Threading; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using System.Collections.Immutable; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using System.Threading; +using MongoDB.Driver; namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors; @@ -159,14 +160,14 @@ public sealed class CiscoCsafConnectorTests { public VexConnectorState? CurrentState { get; private set; } - public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) - => ValueTask.FromResult(CurrentState); - - public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) - { - CurrentState = state; - return ValueTask.CompletedTask; - } + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(CurrentState); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + CurrentState = state; + return ValueTask.CompletedTask; + } } private sealed class InMemoryRawSink : IVexRawDocumentSink diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs index c28dd4e1..66073403 100644 --- a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs @@ -10,11 +10,12 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.MSRC.CSAF; -using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; -using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Storage.Mongo; -using Xunit; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using Xunit; +using MongoDB.Driver; namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Connectors; @@ -316,14 +317,14 @@ public sealed class MsrcCsafConnectorTests { public VexConnectorState? State { get; private set; } - public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) - => ValueTask.FromResult(State); - - public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) - { - State = state; - return ValueTask.CompletedTask; - } + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(State); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + State = state; + return ValueTask.CompletedTask; + } } private sealed class TestHttpMessageHandler : HttpMessageHandler diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/MsrcCsafConnector.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/MsrcCsafConnector.cs index 21e558e8..414f2999 100644 --- a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/MsrcCsafConnector.cs +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/MsrcCsafConnector.cs @@ -1,581 +1,581 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Runtime.CompilerServices; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; -using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Storage.Mongo; - -namespace StellaOps.Excititor.Connectors.MSRC.CSAF; - -public sealed class MsrcCsafConnector : VexConnectorBase -{ - private const string QuarantineMetadataKey = "excititor.quarantine.reason"; - private const string FormatMetadataKey = "msrc.csaf.format"; - private const string VulnerabilityMetadataKey = "msrc.vulnerabilityId"; - private const string AdvisoryIdMetadataKey = "msrc.advisoryId"; - private const string LastModifiedMetadataKey = "msrc.lastModified"; - private const string ReleaseDateMetadataKey = "msrc.releaseDate"; - private const string CvssSeverityMetadataKey = "msrc.severity"; - private const string CvrfUrlMetadataKey = "msrc.cvrfUrl"; - - private static readonly VexConnectorDescriptor DescriptorInstance = new( - id: "excititor:msrc", - kind: VexProviderKind.Vendor, - displayName: "Microsoft MSRC CSAF") - { - Description = "Authenticated connector for Microsoft Security Response Center CSAF advisories.", - SupportedFormats = ImmutableArray.Create(VexDocumentFormat.Csaf), - Tags = ImmutableArray.Create("microsoft", "csaf", "vendor"), - }; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMsrcTokenProvider _tokenProvider; - private readonly IVexConnectorStateRepository _stateRepository; - private readonly IOptions _options; - private readonly ILogger _logger; - private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - ReadCommentHandling = JsonCommentHandling.Skip, - }; - - private MsrcConnectorOptions? _validatedOptions; - - public MsrcCsafConnector( - IHttpClientFactory httpClientFactory, - IMsrcTokenProvider tokenProvider, - IVexConnectorStateRepository stateRepository, - IOptions options, - ILogger logger, - TimeProvider timeProvider) - : base(DescriptorInstance, logger, timeProvider) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = options ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) - { - var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered."); - options.Validate(); - _validatedOptions = options; - - LogConnectorEvent( - LogLevel.Information, - "validate", - "Validated MSRC CSAF connector options.", - new Dictionary - { - ["baseUri"] = options.BaseUri.ToString(), - ["locale"] = options.Locale, - ["apiVersion"] = options.ApiVersion, - ["pageSize"] = options.PageSize, - ["maxAdvisories"] = options.MaxAdvisoriesPerFetch, - }); - - return ValueTask.CompletedTask; - } - - public override async IAsyncEnumerable FetchAsync( - VexConnectorContext context, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - var options = EnsureOptionsValidated(); - var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); - var (from, to) = CalculateWindow(context.Since, state, options); - - LogConnectorEvent( - LogLevel.Information, - "fetch.window", - $"Fetching MSRC CSAF advisories updated between {from:O} and {to:O}.", - new Dictionary - { - ["from"] = from, - ["to"] = to, - ["cursorOverlapSeconds"] = options.CursorOverlap.TotalSeconds, - }); - - var client = await CreateAuthenticatedClientAsync(options, cancellationToken).ConfigureAwait(false); - - var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; - var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); - var digestList = new List(knownDigests); - var latest = state?.LastUpdated ?? from; - var fetched = 0; - var stateChanged = false; - - await foreach (var summary in EnumerateSummariesAsync(client, options, from, to, cancellationToken).ConfigureAwait(false)) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (fetched >= options.MaxAdvisoriesPerFetch) - { - break; - } - - if (string.IsNullOrWhiteSpace(summary.CvrfUrl)) - { - LogConnectorEvent(LogLevel.Debug, "skip.no-cvrf", $"Skipping MSRC advisory {summary.Id} because no CSAF URL was provided."); - continue; - } - - var documentUri = ResolveCvrfUri(options.BaseUri, summary.CvrfUrl); - - VexRawDocument? rawDocument = null; - try - { - rawDocument = await DownloadCsafAsync(client, summary, documentUri, options, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - LogConnectorEvent(LogLevel.Warning, "fetch.error", $"Failed to download MSRC CSAF package {documentUri}.", new Dictionary - { - ["advisoryId"] = summary.Id, - ["vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id, - }, ex); - - await Task.Delay(GetRetryDelay(options, 1), cancellationToken).ConfigureAwait(false); - continue; - } - - if (!digestSet.Add(rawDocument.Digest)) - { - LogConnectorEvent(LogLevel.Debug, "skip.duplicate", $"Skipping MSRC CSAF package {documentUri} because it was already processed."); - continue; - } - - await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); - digestList.Add(rawDocument.Digest); - stateChanged = true; - fetched++; - - latest = DetermineLatest(summary, latest) ?? latest; - - var quarantineReason = rawDocument.Metadata.TryGetValue(QuarantineMetadataKey, out var reason) ? reason : null; - if (quarantineReason is not null) - { - LogConnectorEvent(LogLevel.Warning, "quarantine", $"Quarantined MSRC CSAF package {documentUri} ({quarantineReason})."); - continue; - } - - yield return rawDocument; - - if (options.RequestDelay > TimeSpan.Zero) - { - await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false); - } - } - - if (stateChanged) - { - if (digestList.Count > options.MaxTrackedDigests) - { - var trimmed = digestList.Count - options.MaxTrackedDigests; - digestList.RemoveRange(0, trimmed); - } - - var baseState = state ?? new VexConnectorState( - Descriptor.Id, - null, - ImmutableArray.Empty, - ImmutableDictionary.Empty, - null, - 0, - null, - null); - var newState = baseState with - { - LastUpdated = latest == DateTimeOffset.MinValue ? state?.LastUpdated : latest, - DocumentDigests = digestList.ToImmutableArray(), - }; - - await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); - } - - LogConnectorEvent( - LogLevel.Information, - "fetch.completed", - $"MSRC CSAF fetch completed with {fetched} new documents.", - new Dictionary - { - ["fetched"] = fetched, - ["stateChanged"] = stateChanged, - ["lastUpdated"] = latest, - }); - } - - public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => throw new NotSupportedException("MSRC CSAF connector relies on CSAF normalizers for document processing."); - - private async Task DownloadCsafAsync( - HttpClient client, - MsrcVulnerabilitySummary summary, - Uri documentUri, - MsrcConnectorOptions options, - CancellationToken cancellationToken) - { - using var response = await SendWithRetryAsync( - client, - () => new HttpRequestMessage(HttpMethod.Get, documentUri), - options, - cancellationToken).ConfigureAwait(false); - - var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); - - var validation = ValidateCsafPayload(payload); - var metadata = BuildMetadata(builder => - { - builder.Add(AdvisoryIdMetadataKey, summary.Id); - builder.Add(VulnerabilityMetadataKey, summary.VulnerabilityId ?? summary.Id); - builder.Add(CvrfUrlMetadataKey, documentUri.ToString()); - builder.Add(FormatMetadataKey, validation.Format); - - if (!string.IsNullOrWhiteSpace(summary.Severity)) - { - builder.Add(CvssSeverityMetadataKey, summary.Severity); - } - - if (summary.LastModifiedDate is not null) - { - builder.Add(LastModifiedMetadataKey, summary.LastModifiedDate.Value.ToString("O")); - } - - if (summary.ReleaseDate is not null) - { - builder.Add(ReleaseDateMetadataKey, summary.ReleaseDate.Value.ToString("O")); - } - - if (!string.IsNullOrWhiteSpace(validation.QuarantineReason)) - { - builder.Add(QuarantineMetadataKey, validation.QuarantineReason); - } - - if (response.Headers.ETag is not null) - { - builder.Add("http.etag", response.Headers.ETag.Tag); - } - - if (response.Content.Headers.LastModified is { } lastModified) - { - builder.Add("http.lastModified", lastModified.ToString("O")); - } - }); - - return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, payload, metadata); - } - - private async Task CreateAuthenticatedClientAsync(MsrcConnectorOptions options, CancellationToken cancellationToken) - { - var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false); - var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.ApiClientName); - - client.DefaultRequestHeaders.Remove("Authorization"); - client.DefaultRequestHeaders.Add("Authorization", $"{token.Type} {token.Value}"); - client.DefaultRequestHeaders.Remove("Accept-Language"); - client.DefaultRequestHeaders.Add("Accept-Language", options.Locale); - client.DefaultRequestHeaders.Remove("api-version"); - client.DefaultRequestHeaders.Add("api-version", options.ApiVersion); - client.DefaultRequestHeaders.Remove("Accept"); - client.DefaultRequestHeaders.Add("Accept", "application/json"); - - return client; - } - - private async Task SendWithRetryAsync( - HttpClient client, - Func requestFactory, - MsrcConnectorOptions options, - CancellationToken cancellationToken) - { - Exception? lastError = null; - HttpResponseMessage? response = null; - - for (var attempt = 1; attempt <= options.MaxRetryAttempts; attempt++) - { - response?.Dispose(); - using var request = requestFactory(); - try - { - response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - - if (response.IsSuccessStatusCode) - { - return response; - } - - if (!ShouldRetry(response.StatusCode) || attempt == options.MaxRetryAttempts) - { - response.EnsureSuccessStatusCode(); - } - } - catch (Exception ex) when (IsTransient(ex) && attempt < options.MaxRetryAttempts) - { - lastError = ex; - LogConnectorEvent(LogLevel.Warning, "retry", $"Retrying MSRC request (attempt {attempt}/{options.MaxRetryAttempts}).", exception: ex); - } - catch (Exception ex) - { - response?.Dispose(); - throw; - } - - await Task.Delay(GetRetryDelay(options, attempt), cancellationToken).ConfigureAwait(false); - } - - response?.Dispose(); - throw lastError ?? new InvalidOperationException("MSRC request retries exhausted."); - } - - private TimeSpan GetRetryDelay(MsrcConnectorOptions options, int attempt) - { - var baseDelay = options.RetryBaseDelay.TotalMilliseconds; - var multiplier = Math.Pow(2, Math.Max(0, attempt - 1)); - var jitter = Random.Shared.NextDouble() * baseDelay * 0.25; - var delayMs = Math.Min(baseDelay * multiplier + jitter, TimeSpan.FromMinutes(5).TotalMilliseconds); - return TimeSpan.FromMilliseconds(delayMs); - } - - private async IAsyncEnumerable EnumerateSummariesAsync( - HttpClient client, - MsrcConnectorOptions options, - DateTimeOffset from, - DateTimeOffset to, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - var fetched = 0; - var requestUri = BuildSummaryUri(options, from, to); - - while (requestUri is not null && fetched < options.MaxAdvisoriesPerFetch) - { - using var response = await SendWithRetryAsync( - client, - () => new HttpRequestMessage(HttpMethod.Get, requestUri), - options, - cancellationToken).ConfigureAwait(false); - - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - var payload = await JsonSerializer.DeserializeAsync(stream, _serializerOptions, cancellationToken).ConfigureAwait(false) - ?? new MsrcSummaryResponse(); - - foreach (var summary in payload.Value) - { - if (string.IsNullOrWhiteSpace(summary.CvrfUrl)) - { - continue; - } - - yield return summary; - fetched++; - - if (fetched >= options.MaxAdvisoriesPerFetch) - { - yield break; - } - } - - if (string.IsNullOrWhiteSpace(payload.NextLink)) - { - break; - } - - if (!Uri.TryCreate(payload.NextLink, UriKind.Absolute, out requestUri)) - { - LogConnectorEvent(LogLevel.Warning, "pagination.invalid", $"MSRC pagination returned invalid next link '{payload.NextLink}'."); - break; - } - } - } - - private static Uri BuildSummaryUri(MsrcConnectorOptions options, DateTimeOffset from, DateTimeOffset to) - { - var baseText = options.BaseUri.ToString().TrimEnd('/'); - var builder = new StringBuilder(baseText.Length + 128); - builder.Append(baseText); - if (!baseText.EndsWith("/vulnerabilities", StringComparison.OrdinalIgnoreCase)) - { - builder.Append("/vulnerabilities"); - } - - builder.Append("?"); - builder.Append("$top=").Append(options.PageSize); - builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(from.ToUniversalTime().ToString("O"))); - builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(to.ToUniversalTime().ToString("O"))); - builder.Append("&$orderby=lastModifiedDate"); - builder.Append("&locale=").Append(Uri.EscapeDataString(options.Locale)); - builder.Append("&api-version=").Append(Uri.EscapeDataString(options.ApiVersion)); - - return new Uri(builder.ToString(), UriKind.Absolute); - } - - private (DateTimeOffset From, DateTimeOffset To) CalculateWindow( - DateTimeOffset? contextSince, - VexConnectorState? state, - MsrcConnectorOptions options) - { - var now = UtcNow(); - var since = contextSince ?? state?.LastUpdated ?? options.InitialLastModified ?? now.AddDays(-30); - - if (state?.LastUpdated is { } persisted && persisted > since) - { - since = persisted; - } - - if (options.CursorOverlap > TimeSpan.Zero) - { - since = since.Add(-options.CursorOverlap); - } - - if (since < now.AddYears(-20)) - { - since = now.AddYears(-20); - } - - return (since, now); - } - - private static bool ShouldRetry(HttpStatusCode statusCode) - => statusCode == HttpStatusCode.TooManyRequests || - (int)statusCode >= 500; - - private static bool IsTransient(Exception exception) - => exception is HttpRequestException or IOException or TaskCanceledException; - - private static Uri ResolveCvrfUri(Uri baseUri, string cvrfUrl) - => Uri.TryCreate(cvrfUrl, UriKind.Absolute, out var absolute) - ? absolute - : new Uri(baseUri, cvrfUrl); - - private static CsafValidationResult ValidateCsafPayload(ReadOnlyMemory payload) - { - try - { - if (IsZip(payload.Span)) - { - using var zipStream = new MemoryStream(payload.ToArray(), writable: false); - using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read, leaveOpen: true); - var entry = archive.Entries.FirstOrDefault(e => e.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) - ?? archive.Entries.FirstOrDefault(); - if (entry is null) - { - return new CsafValidationResult("zip", "Zip archive did not contain any entries."); - } - - using var entryStream = entry.Open(); - using var reader = new StreamReader(entryStream, Encoding.UTF8); - using var json = JsonDocument.Parse(reader.ReadToEnd()); - return CsafValidationResult.Valid("zip"); - } - - if (IsGzip(payload.Span)) - { - using var input = new MemoryStream(payload.ToArray(), writable: false); - using var gzip = new GZipStream(input, CompressionMode.Decompress); - using var reader = new StreamReader(gzip, Encoding.UTF8); - using var json = JsonDocument.Parse(reader.ReadToEnd()); - return CsafValidationResult.Valid("gzip"); - } - - using var jsonDocument = JsonDocument.Parse(payload.Span); - return CsafValidationResult.Valid("json"); - } - catch (JsonException ex) - { - return new CsafValidationResult("json", $"JSON parse failed: {ex.Message}"); - } - catch (InvalidDataException ex) - { - return new CsafValidationResult("invalid", ex.Message); - } - catch (EndOfStreamException ex) - { - return new CsafValidationResult("invalid", ex.Message); - } - } - - private static bool IsZip(ReadOnlySpan content) - => content.Length > 3 && content[0] == 0x50 && content[1] == 0x4B; - - private static bool IsGzip(ReadOnlySpan content) - => content.Length > 2 && content[0] == 0x1F && content[1] == 0x8B; - - private static DateTimeOffset? DetermineLatest(MsrcVulnerabilitySummary summary, DateTimeOffset? current) - { - var candidate = summary.LastModifiedDate ?? summary.ReleaseDate; - if (candidate is null) - { - return current; - } - - if (current is null || candidate > current) - { - return candidate; - } - - return current; - } - - private MsrcConnectorOptions EnsureOptionsValidated() - { - if (_validatedOptions is not null) - { - return _validatedOptions; - } - - var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered."); - options.Validate(); - _validatedOptions = options; - return options; - } - - private sealed record CsafValidationResult(string Format, string? QuarantineReason) - { - public static CsafValidationResult Valid(string format) => new(format, null); - } -} - -internal sealed record MsrcSummaryResponse -{ - [JsonPropertyName("value")] - public List Value { get; init; } = new(); - - [JsonPropertyName("@odata.nextLink")] - public string? NextLink { get; init; } -} - -internal sealed record MsrcVulnerabilitySummary -{ - [JsonPropertyName("id")] - public string Id { get; init; } = string.Empty; - - [JsonPropertyName("vulnerabilityId")] - public string? VulnerabilityId { get; init; } - - [JsonPropertyName("severity")] - public string? Severity { get; init; } - - [JsonPropertyName("releaseDate")] - public DateTimeOffset? ReleaseDate { get; init; } - - [JsonPropertyName("lastModifiedDate")] - public DateTimeOffset? LastModifiedDate { get; init; } - - [JsonPropertyName("cvrfUrl")] - public string? CvrfUrl { get; init; } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.MSRC.CSAF; + +public sealed class MsrcCsafConnector : VexConnectorBase +{ + private const string QuarantineMetadataKey = "excititor.quarantine.reason"; + private const string FormatMetadataKey = "msrc.csaf.format"; + private const string VulnerabilityMetadataKey = "msrc.vulnerabilityId"; + private const string AdvisoryIdMetadataKey = "msrc.advisoryId"; + private const string LastModifiedMetadataKey = "msrc.lastModified"; + private const string ReleaseDateMetadataKey = "msrc.releaseDate"; + private const string CvssSeverityMetadataKey = "msrc.severity"; + private const string CvrfUrlMetadataKey = "msrc.cvrfUrl"; + + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "excititor:msrc", + kind: VexProviderKind.Vendor, + displayName: "Microsoft MSRC CSAF") + { + Description = "Authenticated connector for Microsoft Security Response Center CSAF advisories.", + SupportedFormats = ImmutableArray.Create(VexDocumentFormat.Csaf), + Tags = ImmutableArray.Create("microsoft", "csaf", "vendor"), + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMsrcTokenProvider _tokenProvider; + private readonly IVexConnectorStateRepository _stateRepository; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + + private MsrcConnectorOptions? _validatedOptions; + + public MsrcCsafConnector( + IHttpClientFactory httpClientFactory, + IMsrcTokenProvider tokenProvider, + IVexConnectorStateRepository stateRepository, + IOptions options, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered."); + options.Validate(); + _validatedOptions = options; + + LogConnectorEvent( + LogLevel.Information, + "validate", + "Validated MSRC CSAF connector options.", + new Dictionary + { + ["baseUri"] = options.BaseUri.ToString(), + ["locale"] = options.Locale, + ["apiVersion"] = options.ApiVersion, + ["pageSize"] = options.PageSize, + ["maxAdvisories"] = options.MaxAdvisoriesPerFetch, + }); + + return ValueTask.CompletedTask; + } + + public override async IAsyncEnumerable FetchAsync( + VexConnectorContext context, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var options = EnsureOptionsValidated(); + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + var (from, to) = CalculateWindow(context.Since, state, options); + + LogConnectorEvent( + LogLevel.Information, + "fetch.window", + $"Fetching MSRC CSAF advisories updated between {from:O} and {to:O}.", + new Dictionary + { + ["from"] = from, + ["to"] = to, + ["cursorOverlapSeconds"] = options.CursorOverlap.TotalSeconds, + }); + + var client = await CreateAuthenticatedClientAsync(options, cancellationToken).ConfigureAwait(false); + + var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); + var digestList = new List(knownDigests); + var latest = state?.LastUpdated ?? from; + var fetched = 0; + var stateChanged = false; + + await foreach (var summary in EnumerateSummariesAsync(client, options, from, to, cancellationToken).ConfigureAwait(false)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (fetched >= options.MaxAdvisoriesPerFetch) + { + break; + } + + if (string.IsNullOrWhiteSpace(summary.CvrfUrl)) + { + LogConnectorEvent(LogLevel.Debug, "skip.no-cvrf", $"Skipping MSRC advisory {summary.Id} because no CSAF URL was provided."); + continue; + } + + var documentUri = ResolveCvrfUri(options.BaseUri, summary.CvrfUrl); + + VexRawDocument? rawDocument = null; + try + { + rawDocument = await DownloadCsafAsync(client, summary, documentUri, options, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectorEvent(LogLevel.Warning, "fetch.error", $"Failed to download MSRC CSAF package {documentUri}.", new Dictionary + { + ["advisoryId"] = summary.Id, + ["vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id, + }, ex); + + await Task.Delay(GetRetryDelay(options, 1), cancellationToken).ConfigureAwait(false); + continue; + } + + if (!digestSet.Add(rawDocument.Digest)) + { + LogConnectorEvent(LogLevel.Debug, "skip.duplicate", $"Skipping MSRC CSAF package {documentUri} because it was already processed."); + continue; + } + + await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); + digestList.Add(rawDocument.Digest); + stateChanged = true; + fetched++; + + latest = DetermineLatest(summary, latest) ?? latest; + + var quarantineReason = rawDocument.Metadata.TryGetValue(QuarantineMetadataKey, out var reason) ? reason : null; + if (quarantineReason is not null) + { + LogConnectorEvent(LogLevel.Warning, "quarantine", $"Quarantined MSRC CSAF package {documentUri} ({quarantineReason})."); + continue; + } + + yield return rawDocument; + + if (options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + if (stateChanged) + { + if (digestList.Count > options.MaxTrackedDigests) + { + var trimmed = digestList.Count - options.MaxTrackedDigests; + digestList.RemoveRange(0, trimmed); + } + + var baseState = state ?? new VexConnectorState( + Descriptor.Id, + null, + ImmutableArray.Empty, + ImmutableDictionary.Empty, + null, + 0, + null, + null); + var newState = baseState with + { + LastUpdated = latest == DateTimeOffset.MinValue ? state?.LastUpdated : latest, + DocumentDigests = digestList.ToImmutableArray(), + }; + + await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); + } + + LogConnectorEvent( + LogLevel.Information, + "fetch.completed", + $"MSRC CSAF fetch completed with {fetched} new documents.", + new Dictionary + { + ["fetched"] = fetched, + ["stateChanged"] = stateChanged, + ["lastUpdated"] = latest, + }); + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("MSRC CSAF connector relies on CSAF normalizers for document processing."); + + private async Task DownloadCsafAsync( + HttpClient client, + MsrcVulnerabilitySummary summary, + Uri documentUri, + MsrcConnectorOptions options, + CancellationToken cancellationToken) + { + using var response = await SendWithRetryAsync( + client, + () => new HttpRequestMessage(HttpMethod.Get, documentUri), + options, + cancellationToken).ConfigureAwait(false); + + var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + + var validation = ValidateCsafPayload(payload); + var metadata = BuildMetadata(builder => + { + builder.Add(AdvisoryIdMetadataKey, summary.Id); + builder.Add(VulnerabilityMetadataKey, summary.VulnerabilityId ?? summary.Id); + builder.Add(CvrfUrlMetadataKey, documentUri.ToString()); + builder.Add(FormatMetadataKey, validation.Format); + + if (!string.IsNullOrWhiteSpace(summary.Severity)) + { + builder.Add(CvssSeverityMetadataKey, summary.Severity); + } + + if (summary.LastModifiedDate is not null) + { + builder.Add(LastModifiedMetadataKey, summary.LastModifiedDate.Value.ToString("O")); + } + + if (summary.ReleaseDate is not null) + { + builder.Add(ReleaseDateMetadataKey, summary.ReleaseDate.Value.ToString("O")); + } + + if (!string.IsNullOrWhiteSpace(validation.QuarantineReason)) + { + builder.Add(QuarantineMetadataKey, validation.QuarantineReason); + } + + if (response.Headers.ETag is not null) + { + builder.Add("http.etag", response.Headers.ETag.Tag); + } + + if (response.Content.Headers.LastModified is { } lastModified) + { + builder.Add("http.lastModified", lastModified.ToString("O")); + } + }); + + return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, payload, metadata); + } + + private async Task CreateAuthenticatedClientAsync(MsrcConnectorOptions options, CancellationToken cancellationToken) + { + var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false); + var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.ApiClientName); + + client.DefaultRequestHeaders.Remove("Authorization"); + client.DefaultRequestHeaders.Add("Authorization", $"{token.Type} {token.Value}"); + client.DefaultRequestHeaders.Remove("Accept-Language"); + client.DefaultRequestHeaders.Add("Accept-Language", options.Locale); + client.DefaultRequestHeaders.Remove("api-version"); + client.DefaultRequestHeaders.Add("api-version", options.ApiVersion); + client.DefaultRequestHeaders.Remove("Accept"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + + return client; + } + + private async Task SendWithRetryAsync( + HttpClient client, + Func requestFactory, + MsrcConnectorOptions options, + CancellationToken cancellationToken) + { + Exception? lastError = null; + HttpResponseMessage? response = null; + + for (var attempt = 1; attempt <= options.MaxRetryAttempts; attempt++) + { + response?.Dispose(); + using var request = requestFactory(); + try + { + response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.IsSuccessStatusCode) + { + return response; + } + + if (!ShouldRetry(response.StatusCode) || attempt == options.MaxRetryAttempts) + { + response.EnsureSuccessStatusCode(); + } + } + catch (Exception ex) when (IsTransient(ex) && attempt < options.MaxRetryAttempts) + { + lastError = ex; + LogConnectorEvent(LogLevel.Warning, "retry", $"Retrying MSRC request (attempt {attempt}/{options.MaxRetryAttempts}).", exception: ex); + } + catch (Exception) + { + response?.Dispose(); + throw; + } + + await Task.Delay(GetRetryDelay(options, attempt), cancellationToken).ConfigureAwait(false); + } + + response?.Dispose(); + throw lastError ?? new InvalidOperationException("MSRC request retries exhausted."); + } + + private TimeSpan GetRetryDelay(MsrcConnectorOptions options, int attempt) + { + var baseDelay = options.RetryBaseDelay.TotalMilliseconds; + var multiplier = Math.Pow(2, Math.Max(0, attempt - 1)); + var jitter = Random.Shared.NextDouble() * baseDelay * 0.25; + var delayMs = Math.Min(baseDelay * multiplier + jitter, TimeSpan.FromMinutes(5).TotalMilliseconds); + return TimeSpan.FromMilliseconds(delayMs); + } + + private async IAsyncEnumerable EnumerateSummariesAsync( + HttpClient client, + MsrcConnectorOptions options, + DateTimeOffset from, + DateTimeOffset to, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var fetched = 0; + var requestUri = BuildSummaryUri(options, from, to); + + while (requestUri is not null && fetched < options.MaxAdvisoriesPerFetch) + { + using var response = await SendWithRetryAsync( + client, + () => new HttpRequestMessage(HttpMethod.Get, requestUri), + options, + cancellationToken).ConfigureAwait(false); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var payload = await JsonSerializer.DeserializeAsync(stream, _serializerOptions, cancellationToken).ConfigureAwait(false) + ?? new MsrcSummaryResponse(); + + foreach (var summary in payload.Value) + { + if (string.IsNullOrWhiteSpace(summary.CvrfUrl)) + { + continue; + } + + yield return summary; + fetched++; + + if (fetched >= options.MaxAdvisoriesPerFetch) + { + yield break; + } + } + + if (string.IsNullOrWhiteSpace(payload.NextLink)) + { + break; + } + + if (!Uri.TryCreate(payload.NextLink, UriKind.Absolute, out requestUri)) + { + LogConnectorEvent(LogLevel.Warning, "pagination.invalid", $"MSRC pagination returned invalid next link '{payload.NextLink}'."); + break; + } + } + } + + private static Uri BuildSummaryUri(MsrcConnectorOptions options, DateTimeOffset from, DateTimeOffset to) + { + var baseText = options.BaseUri.ToString().TrimEnd('/'); + var builder = new StringBuilder(baseText.Length + 128); + builder.Append(baseText); + if (!baseText.EndsWith("/vulnerabilities", StringComparison.OrdinalIgnoreCase)) + { + builder.Append("/vulnerabilities"); + } + + builder.Append("?"); + builder.Append("$top=").Append(options.PageSize); + builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(from.ToUniversalTime().ToString("O"))); + builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(to.ToUniversalTime().ToString("O"))); + builder.Append("&$orderby=lastModifiedDate"); + builder.Append("&locale=").Append(Uri.EscapeDataString(options.Locale)); + builder.Append("&api-version=").Append(Uri.EscapeDataString(options.ApiVersion)); + + return new Uri(builder.ToString(), UriKind.Absolute); + } + + private (DateTimeOffset From, DateTimeOffset To) CalculateWindow( + DateTimeOffset? contextSince, + VexConnectorState? state, + MsrcConnectorOptions options) + { + var now = UtcNow(); + var since = contextSince ?? state?.LastUpdated ?? options.InitialLastModified ?? now.AddDays(-30); + + if (state?.LastUpdated is { } persisted && persisted > since) + { + since = persisted; + } + + if (options.CursorOverlap > TimeSpan.Zero) + { + since = since.Add(-options.CursorOverlap); + } + + if (since < now.AddYears(-20)) + { + since = now.AddYears(-20); + } + + return (since, now); + } + + private static bool ShouldRetry(HttpStatusCode statusCode) + => statusCode == HttpStatusCode.TooManyRequests || + (int)statusCode >= 500; + + private static bool IsTransient(Exception exception) + => exception is HttpRequestException or IOException or TaskCanceledException; + + private static Uri ResolveCvrfUri(Uri baseUri, string cvrfUrl) + => Uri.TryCreate(cvrfUrl, UriKind.Absolute, out var absolute) + ? absolute + : new Uri(baseUri, cvrfUrl); + + private static CsafValidationResult ValidateCsafPayload(ReadOnlyMemory payload) + { + try + { + if (IsZip(payload.Span)) + { + using var zipStream = new MemoryStream(payload.ToArray(), writable: false); + using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read, leaveOpen: true); + var entry = archive.Entries.FirstOrDefault(e => e.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + ?? archive.Entries.FirstOrDefault(); + if (entry is null) + { + return new CsafValidationResult("zip", "Zip archive did not contain any entries."); + } + + using var entryStream = entry.Open(); + using var reader = new StreamReader(entryStream, Encoding.UTF8); + using var json = JsonDocument.Parse(reader.ReadToEnd()); + return CsafValidationResult.Valid("zip"); + } + + if (IsGzip(payload.Span)) + { + using var input = new MemoryStream(payload.ToArray(), writable: false); + using var gzip = new GZipStream(input, CompressionMode.Decompress); + using var reader = new StreamReader(gzip, Encoding.UTF8); + using var json = JsonDocument.Parse(reader.ReadToEnd()); + return CsafValidationResult.Valid("gzip"); + } + + using var jsonDocument = JsonDocument.Parse(payload); + return CsafValidationResult.Valid("json"); + } + catch (JsonException ex) + { + return new CsafValidationResult("json", $"JSON parse failed: {ex.Message}"); + } + catch (InvalidDataException ex) + { + return new CsafValidationResult("invalid", ex.Message); + } + catch (EndOfStreamException ex) + { + return new CsafValidationResult("invalid", ex.Message); + } + } + + private static bool IsZip(ReadOnlySpan content) + => content.Length > 3 && content[0] == 0x50 && content[1] == 0x4B; + + private static bool IsGzip(ReadOnlySpan content) + => content.Length > 2 && content[0] == 0x1F && content[1] == 0x8B; + + private static DateTimeOffset? DetermineLatest(MsrcVulnerabilitySummary summary, DateTimeOffset? current) + { + var candidate = summary.LastModifiedDate ?? summary.ReleaseDate; + if (candidate is null) + { + return current; + } + + if (current is null || candidate > current) + { + return candidate; + } + + return current; + } + + private MsrcConnectorOptions EnsureOptionsValidated() + { + if (_validatedOptions is not null) + { + return _validatedOptions; + } + + var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered."); + options.Validate(); + _validatedOptions = options; + return options; + } + + private sealed record CsafValidationResult(string Format, string? QuarantineReason) + { + public static CsafValidationResult Valid(string format) => new(format, null); + } +} + +internal sealed record MsrcSummaryResponse +{ + [JsonPropertyName("value")] + public List Value { get; init; } = new(); + + [JsonPropertyName("@odata.nextLink")] + public string? NextLink { get; init; } +} + +internal sealed record MsrcVulnerabilitySummary +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("vulnerabilityId")] + public string? VulnerabilityId { get; init; } + + [JsonPropertyName("severity")] + public string? Severity { get; init; } + + [JsonPropertyName("releaseDate")] + public DateTimeOffset? ReleaseDate { get; init; } + + [JsonPropertyName("lastModifiedDate")] + public DateTimeOffset? LastModifiedDate { get; init; } + + [JsonPropertyName("cvrfUrl")] + public string? CvrfUrl { get; init; } +} diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs index 0b8155bf..2f28532b 100644 --- a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs @@ -15,11 +15,12 @@ using Microsoft.Extensions.Logging.Abstractions; using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.Oracle.CSAF; using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; -using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Storage.Mongo; -using System.IO.Abstractions.TestingHelpers; -using Xunit; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using MongoDB.Driver; namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Connectors; @@ -254,14 +255,14 @@ public sealed class OracleCsafConnectorTests { public VexConnectorState? State { get; private set; } - public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) - => ValueTask.FromResult(State); - - public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) - { - State = state; - return ValueTask.CompletedTask; - } + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(State); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + State = state; + return ValueTask.CompletedTask; + } } private sealed class InMemoryRawSink : IVexRawDocumentSink diff --git a/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs index 46ecd46e..e73d03cb 100644 --- a/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs @@ -10,9 +10,10 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; -using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using MongoDB.Driver; namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.Connectors; @@ -258,20 +259,20 @@ public sealed class RedHatCsafConnectorTests { public VexConnectorState? State { get; private set; } - public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) - { - if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase)) - { - return ValueTask.FromResult(State); - } - - return ValueTask.FromResult(null); - } - - public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) - { - State = state; - return ValueTask.CompletedTask; - } + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase)) + { + return ValueTask.FromResult(State); + } + + return ValueTask.FromResult(null); + } + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + State = state; + return ValueTask.CompletedTask; + } } } diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventClient.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventClient.cs index cf94f5eb..6e1d3565 100644 --- a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventClient.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventClient.cs @@ -14,9 +14,9 @@ using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; -namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; - -internal sealed class RancherHubEventClient +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; + +public sealed class RancherHubEventClient { private readonly IHttpClientFactory _httpClientFactory; private readonly RancherHubTokenProvider _tokenProvider; diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventModels.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventModels.cs index d1434a3e..e6bccf67 100644 --- a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventModels.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventModels.cs @@ -1,21 +1,21 @@ -using System; -using System.Collections.Immutable; - -namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; - -internal sealed record RancherHubEventRecord( - string RawJson, - string? Id, - string? Type, - string? Channel, - DateTimeOffset? PublishedAt, - Uri? DocumentUri, - string? DocumentDigest, - string? DocumentFormat); - -internal sealed record RancherHubEventBatch( - string? Cursor, - string? NextCursor, - ImmutableArray Events, - bool FromOfflineSnapshot, - string RawPayload); +using System; +using System.Collections.Immutable; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; + +public sealed record RancherHubEventRecord( + string RawJson, + string? Id, + string? Type, + string? Channel, + DateTimeOffset? PublishedAt, + Uri? DocumentUri, + string? DocumentDigest, + string? DocumentFormat); + +public sealed record RancherHubEventBatch( + string? Cursor, + string? NextCursor, + ImmutableArray Events, + bool FromOfflineSnapshot, + string RawPayload); diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs index bca605af..97689785 100644 --- a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs @@ -1,344 +1,345 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Security.Cryptography; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; -using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; -using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; -using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; -using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State; -using StellaOps.Excititor.Core; - -namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub; - -public sealed class RancherHubConnector : VexConnectorBase -{ - private static readonly VexConnectorDescriptor StaticDescriptor = new( - id: "excititor:suse.rancher", - kind: VexProviderKind.Hub, - displayName: "SUSE Rancher VEX Hub") - { - Tags = ImmutableArray.Create("hub", "suse", "offline"), - }; - - private readonly RancherHubMetadataLoader _metadataLoader; - private readonly RancherHubEventClient _eventClient; - private readonly RancherHubCheckpointManager _checkpointManager; - private readonly RancherHubTokenProvider _tokenProvider; - private readonly IHttpClientFactory _httpClientFactory; - private readonly IEnumerable> _validators; - - private RancherHubConnectorOptions? _options; - private RancherHubMetadataResult? _metadata; - - public RancherHubConnector( - RancherHubMetadataLoader metadataLoader, - RancherHubEventClient eventClient, - RancherHubCheckpointManager checkpointManager, - RancherHubTokenProvider tokenProvider, - IHttpClientFactory httpClientFactory, - ILogger logger, - TimeProvider timeProvider, - IEnumerable>? validators = null) - : base(StaticDescriptor, logger, timeProvider) - { - _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); - _eventClient = eventClient ?? throw new ArgumentNullException(nameof(eventClient)); - _checkpointManager = checkpointManager ?? throw new ArgumentNullException(nameof(checkpointManager)); - _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _validators = validators ?? Array.Empty>(); - } - - public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) - { - _options = VexConnectorOptionsBinder.Bind( - Descriptor, - settings, - validators: _validators); - - _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - - LogConnectorEvent(LogLevel.Information, "validate", "Rancher hub discovery loaded.", new Dictionary - { - ["discoveryUri"] = _options.DiscoveryUri.ToString(), - ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), - ["requiresAuth"] = _metadata.Metadata.Subscription.RequiresAuthentication, - ["fromOffline"] = _metadata.FromOfflineSnapshot, - }); - } - - public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - if (_options is null) - { - throw new InvalidOperationException("Connector must be validated before fetch operations."); - } - - if (_metadata is null) - { - _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - } - - var checkpoint = await _checkpointManager.LoadAsync(Descriptor.Id, context, cancellationToken).ConfigureAwait(false); - var digestHistory = checkpoint.Digests.ToList(); - var dedupeSet = new HashSet(checkpoint.Digests, StringComparer.OrdinalIgnoreCase); - var latestCursor = checkpoint.Cursor; - var latestPublishedAt = checkpoint.LastPublishedAt ?? checkpoint.EffectiveSince; - var stateChanged = false; - - LogConnectorEvent(LogLevel.Information, "fetch_start", "Starting Rancher hub event ingestion.", new Dictionary - { - ["since"] = checkpoint.EffectiveSince?.ToString("O"), - ["cursor"] = checkpoint.Cursor, - ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), - ["offline"] = checkpoint.Cursor is null && _options.PreferOfflineSnapshot, - }); - - await foreach (var batch in _eventClient.FetchEventBatchesAsync( - _options, - _metadata.Metadata, - checkpoint.Cursor, - checkpoint.EffectiveSince, - _metadata.Metadata.Subscription.Channels, - cancellationToken).ConfigureAwait(false)) - { - LogConnectorEvent(LogLevel.Debug, "batch", "Processing Rancher hub batch.", new Dictionary - { - ["cursor"] = batch.Cursor, - ["nextCursor"] = batch.NextCursor, - ["count"] = batch.Events.Length, - ["offline"] = batch.FromOfflineSnapshot, - }); - - if (!string.IsNullOrWhiteSpace(batch.NextCursor) && !string.Equals(batch.NextCursor, latestCursor, StringComparison.Ordinal)) - { - latestCursor = batch.NextCursor; - stateChanged = true; - } - else if (string.IsNullOrWhiteSpace(latestCursor) && !string.IsNullOrWhiteSpace(batch.Cursor)) - { - latestCursor = batch.Cursor; - } - - foreach (var record in batch.Events) - { - cancellationToken.ThrowIfCancellationRequested(); - - var result = await ProcessEventAsync(record, batch, context, dedupeSet, digestHistory, cancellationToken).ConfigureAwait(false); - if (result.ProcessedDocument is not null) - { - yield return result.ProcessedDocument; - stateChanged = true; - if (result.PublishedAt is { } published && (latestPublishedAt is null || published > latestPublishedAt)) - { - latestPublishedAt = published; - } - } - else if (result.Quarantined) - { - stateChanged = true; - } - } - } - - if (stateChanged || !string.Equals(latestCursor, checkpoint.Cursor, StringComparison.Ordinal) || latestPublishedAt != checkpoint.LastPublishedAt) - { - await _checkpointManager.SaveAsync( - Descriptor.Id, - latestCursor, - latestPublishedAt, - digestHistory.ToImmutableArray(), - cancellationToken).ConfigureAwait(false); - } - } - - public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => throw new NotSupportedException("RancherHubConnector relies on format-specific normalizers for CSAF/OpenVEX payloads."); - - public RancherHubMetadata? GetCachedMetadata() => _metadata?.Metadata; - - private async Task ProcessEventAsync( - RancherHubEventRecord record, - RancherHubEventBatch batch, - VexConnectorContext context, - HashSet dedupeSet, - List digestHistory, - CancellationToken cancellationToken) - { - var quarantineKey = BuildQuarantineKey(record); - if (dedupeSet.Contains(quarantineKey)) - { - return EventProcessingResult.QuarantinedOnly; - } - - if (record.DocumentUri is null || string.IsNullOrWhiteSpace(record.Id)) - { - await QuarantineAsync(record, batch, "missing documentUri or id", context, cancellationToken).ConfigureAwait(false); - AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); - return EventProcessingResult.QuarantinedOnly; - } - - var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); - using var request = await CreateDocumentRequestAsync(record.DocumentUri, cancellationToken).ConfigureAwait(false); - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - { - await QuarantineAsync(record, batch, $"document fetch failed ({(int)response.StatusCode} {response.StatusCode})", context, cancellationToken).ConfigureAwait(false); - AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); - return EventProcessingResult.QuarantinedOnly; - } - - var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); - var publishedAt = record.PublishedAt ?? UtcNow(); - var metadata = BuildMetadata(builder => builder - .Add("rancher.event.id", record.Id) - .Add("rancher.event.type", record.Type) - .Add("rancher.event.channel", record.Channel) - .Add("rancher.event.published", publishedAt) - .Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor) - .Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false") - .Add("rancher.event.declaredDigest", record.DocumentDigest)); - - var format = ResolveFormat(record.DocumentFormat); - var document = CreateRawDocument(format, record.DocumentUri, contentBytes, metadata); - - if (!string.IsNullOrWhiteSpace(record.DocumentDigest)) - { - var declared = NormalizeDigest(record.DocumentDigest); - var computed = NormalizeDigest(document.Digest); - if (!string.Equals(declared, computed, StringComparison.OrdinalIgnoreCase)) - { - await QuarantineAsync(record, batch, $"digest mismatch (declared {record.DocumentDigest}, computed {document.Digest})", context, cancellationToken).ConfigureAwait(false); - AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); - return EventProcessingResult.QuarantinedOnly; - } - } - - if (!dedupeSet.Add(document.Digest)) - { - return EventProcessingResult.Skipped; - } - - digestHistory.Add(document.Digest); - await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); - return new EventProcessingResult(document, false, publishedAt); - } - - private async Task CreateDocumentRequestAsync(Uri documentUri, CancellationToken cancellationToken) - { - var request = new HttpRequestMessage(HttpMethod.Get, documentUri); - if (_metadata?.Metadata.Subscription.RequiresAuthentication ?? false) - { - var token = await _tokenProvider.GetAccessTokenAsync(_options!, cancellationToken).ConfigureAwait(false); - if (token is not null) - { - var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType; - request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value); - } - } - - return request; - } - - private async Task QuarantineAsync( - RancherHubEventRecord record, - RancherHubEventBatch batch, - string reason, - VexConnectorContext context, - CancellationToken cancellationToken) - { - var metadata = BuildMetadata(builder => builder - .Add("rancher.event.id", record.Id) - .Add("rancher.event.type", record.Type) - .Add("rancher.event.channel", record.Channel) - .Add("rancher.event.quarantine", "true") - .Add("rancher.event.error", reason) - .Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor) - .Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false")); - - var sourceUri = record.DocumentUri ?? _metadata?.Metadata.Subscription.EventsUri ?? _options!.DiscoveryUri; - var payload = Encoding.UTF8.GetBytes(record.RawJson); - var document = CreateRawDocument(VexDocumentFormat.Csaf, sourceUri, payload, metadata); - await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); - - LogConnectorEvent(LogLevel.Warning, "quarantine", "Rancher hub event moved to quarantine.", new Dictionary - { - ["eventId"] = record.Id ?? "(missing)", - ["reason"] = reason, - }); - } - - private static void AddQuarantineDigest(string key, HashSet dedupeSet, List digestHistory) - { - if (dedupeSet.Add(key)) - { - digestHistory.Add(key); - } - } - - private static string BuildQuarantineKey(RancherHubEventRecord record) - { - if (!string.IsNullOrWhiteSpace(record.Id)) - { - return $"quarantine:{record.Id}"; - } - - Span hash = stackalloc byte[32]; - var bytes = Encoding.UTF8.GetBytes(record.RawJson); - if (!SHA256.TryHashData(bytes, hash, out _)) - { - using var sha = SHA256.Create(); - hash = sha.ComputeHash(bytes); - } - - return $"quarantine:{Convert.ToHexString(hash).ToLowerInvariant()}"; - } - - private static string NormalizeDigest(string digest) - { - if (string.IsNullOrWhiteSpace(digest)) - { - return digest; - } - - var trimmed = digest.Trim(); - return trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) - ? trimmed.ToLowerInvariant() - : $"sha256:{trimmed.ToLowerInvariant()}"; - } - - private static VexDocumentFormat ResolveFormat(string? format) - { - if (string.IsNullOrWhiteSpace(format)) - { - return VexDocumentFormat.Csaf; - } - - return format.ToLowerInvariant() switch - { - "csaf" or "csaf_json" or "json" => VexDocumentFormat.Csaf, - "cyclonedx" or "cyclonedx_vex" => VexDocumentFormat.CycloneDx, - "openvex" => VexDocumentFormat.OpenVex, - "oci" or "oci_attestation" or "attestation" => VexDocumentFormat.OciAttestation, - _ => VexDocumentFormat.Csaf, - }; - } - - private sealed record EventProcessingResult(VexRawDocument? ProcessedDocument, bool Quarantined, DateTimeOffset? PublishedAt) - { - public static EventProcessingResult QuarantinedOnly { get; } = new(null, true, null); - - public static EventProcessingResult Skipped { get; } = new(null, false, null); - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub; + +public sealed class RancherHubConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor StaticDescriptor = new( + id: "excititor:suse.rancher", + kind: VexProviderKind.Hub, + displayName: "SUSE Rancher VEX Hub") + { + Tags = ImmutableArray.Create("hub", "suse", "offline"), + }; + + private readonly RancherHubMetadataLoader _metadataLoader; + private readonly RancherHubEventClient _eventClient; + private readonly RancherHubCheckpointManager _checkpointManager; + private readonly RancherHubTokenProvider _tokenProvider; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IEnumerable> _validators; + + private RancherHubConnectorOptions? _options; + private RancherHubMetadataResult? _metadata; + + public RancherHubConnector( + RancherHubMetadataLoader metadataLoader, + RancherHubEventClient eventClient, + RancherHubCheckpointManager checkpointManager, + RancherHubTokenProvider tokenProvider, + IHttpClientFactory httpClientFactory, + ILogger logger, + TimeProvider timeProvider, + IEnumerable>? validators = null) + : base(StaticDescriptor, logger, timeProvider) + { + _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); + _eventClient = eventClient ?? throw new ArgumentNullException(nameof(eventClient)); + _checkpointManager = checkpointManager ?? throw new ArgumentNullException(nameof(checkpointManager)); + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + + LogConnectorEvent(LogLevel.Information, "validate", "Rancher hub discovery loaded.", new Dictionary + { + ["discoveryUri"] = _options.DiscoveryUri.ToString(), + ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), + ["requiresAuth"] = _metadata.Metadata.Subscription.RequiresAuthentication, + ["fromOffline"] = _metadata.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_metadata is null) + { + _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + } + + var checkpoint = await _checkpointManager.LoadAsync(Descriptor.Id, context, cancellationToken).ConfigureAwait(false); + var digestHistory = checkpoint.Digests.ToList(); + var dedupeSet = new HashSet(checkpoint.Digests, StringComparer.OrdinalIgnoreCase); + var latestCursor = checkpoint.Cursor; + var latestPublishedAt = checkpoint.LastPublishedAt ?? checkpoint.EffectiveSince; + var stateChanged = false; + + LogConnectorEvent(LogLevel.Information, "fetch_start", "Starting Rancher hub event ingestion.", new Dictionary + { + ["since"] = checkpoint.EffectiveSince?.ToString("O"), + ["cursor"] = checkpoint.Cursor, + ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), + ["offline"] = checkpoint.Cursor is null && _options.PreferOfflineSnapshot, + }); + + await foreach (var batch in _eventClient.FetchEventBatchesAsync( + _options, + _metadata.Metadata, + checkpoint.Cursor, + checkpoint.EffectiveSince, + _metadata.Metadata.Subscription.Channels, + cancellationToken).ConfigureAwait(false)) + { + LogConnectorEvent(LogLevel.Debug, "batch", "Processing Rancher hub batch.", new Dictionary + { + ["cursor"] = batch.Cursor, + ["nextCursor"] = batch.NextCursor, + ["count"] = batch.Events.Length, + ["offline"] = batch.FromOfflineSnapshot, + }); + + if (!string.IsNullOrWhiteSpace(batch.NextCursor) && !string.Equals(batch.NextCursor, latestCursor, StringComparison.Ordinal)) + { + latestCursor = batch.NextCursor; + stateChanged = true; + } + else if (string.IsNullOrWhiteSpace(latestCursor) && !string.IsNullOrWhiteSpace(batch.Cursor)) + { + latestCursor = batch.Cursor; + } + + foreach (var record in batch.Events) + { + cancellationToken.ThrowIfCancellationRequested(); + + var result = await ProcessEventAsync(record, batch, context, dedupeSet, digestHistory, cancellationToken).ConfigureAwait(false); + if (result.ProcessedDocument is not null) + { + yield return result.ProcessedDocument; + stateChanged = true; + if (result.PublishedAt is { } published && (latestPublishedAt is null || published > latestPublishedAt)) + { + latestPublishedAt = published; + } + } + else if (result.Quarantined) + { + stateChanged = true; + } + } + } + + if (stateChanged || !string.Equals(latestCursor, checkpoint.Cursor, StringComparison.Ordinal) || latestPublishedAt != checkpoint.LastPublishedAt) + { + await _checkpointManager.SaveAsync( + Descriptor.Id, + latestCursor, + latestPublishedAt, + digestHistory.ToImmutableArray(), + cancellationToken).ConfigureAwait(false); + } + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("RancherHubConnector relies on format-specific normalizers for CSAF/OpenVEX payloads."); + + public RancherHubMetadata? GetCachedMetadata() => _metadata?.Metadata; + + private async Task ProcessEventAsync( + RancherHubEventRecord record, + RancherHubEventBatch batch, + VexConnectorContext context, + HashSet dedupeSet, + List digestHistory, + CancellationToken cancellationToken) + { + var quarantineKey = BuildQuarantineKey(record); + if (dedupeSet.Contains(quarantineKey)) + { + return EventProcessingResult.QuarantinedOnly; + } + + if (record.DocumentUri is null || string.IsNullOrWhiteSpace(record.Id)) + { + await QuarantineAsync(record, batch, "missing documentUri or id", context, cancellationToken).ConfigureAwait(false); + AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); + return EventProcessingResult.QuarantinedOnly; + } + + var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); + using var request = await CreateDocumentRequestAsync(record.DocumentUri, cancellationToken).ConfigureAwait(false); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + await QuarantineAsync(record, batch, $"document fetch failed ({(int)response.StatusCode} {response.StatusCode})", context, cancellationToken).ConfigureAwait(false); + AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); + return EventProcessingResult.QuarantinedOnly; + } + + var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var publishedAt = record.PublishedAt ?? UtcNow(); + var metadata = BuildMetadata(builder => builder + .Add("rancher.event.id", record.Id) + .Add("rancher.event.type", record.Type) + .Add("rancher.event.channel", record.Channel) + .Add("rancher.event.published", publishedAt) + .Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor) + .Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false") + .Add("rancher.event.declaredDigest", record.DocumentDigest)); + + var format = ResolveFormat(record.DocumentFormat); + var document = CreateRawDocument(format, record.DocumentUri, contentBytes, metadata); + + if (!string.IsNullOrWhiteSpace(record.DocumentDigest)) + { + var declared = NormalizeDigest(record.DocumentDigest); + var computed = NormalizeDigest(document.Digest); + if (!string.Equals(declared, computed, StringComparison.OrdinalIgnoreCase)) + { + await QuarantineAsync(record, batch, $"digest mismatch (declared {record.DocumentDigest}, computed {document.Digest})", context, cancellationToken).ConfigureAwait(false); + AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); + return EventProcessingResult.QuarantinedOnly; + } + } + + if (!dedupeSet.Add(document.Digest)) + { + return EventProcessingResult.Skipped; + } + + digestHistory.Add(document.Digest); + await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); + return new EventProcessingResult(document, false, publishedAt); + } + + private async Task CreateDocumentRequestAsync(Uri documentUri, CancellationToken cancellationToken) + { + var request = new HttpRequestMessage(HttpMethod.Get, documentUri); + if (_metadata?.Metadata.Subscription.RequiresAuthentication ?? false) + { + var token = await _tokenProvider.GetAccessTokenAsync(_options!, cancellationToken).ConfigureAwait(false); + if (token is not null) + { + var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType; + request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value); + } + } + + return request; + } + + private async Task QuarantineAsync( + RancherHubEventRecord record, + RancherHubEventBatch batch, + string reason, + VexConnectorContext context, + CancellationToken cancellationToken) + { + var metadata = BuildMetadata(builder => builder + .Add("rancher.event.id", record.Id) + .Add("rancher.event.type", record.Type) + .Add("rancher.event.channel", record.Channel) + .Add("rancher.event.quarantine", "true") + .Add("rancher.event.error", reason) + .Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor) + .Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false")); + + var sourceUri = record.DocumentUri ?? _metadata?.Metadata.Subscription.EventsUri ?? _options!.DiscoveryUri; + var payload = Encoding.UTF8.GetBytes(record.RawJson); + var document = CreateRawDocument(VexDocumentFormat.Csaf, sourceUri, payload, metadata); + await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); + + LogConnectorEvent(LogLevel.Warning, "quarantine", "Rancher hub event moved to quarantine.", new Dictionary + { + ["eventId"] = record.Id ?? "(missing)", + ["reason"] = reason, + }); + } + + private static void AddQuarantineDigest(string key, HashSet dedupeSet, List digestHistory) + { + if (dedupeSet.Add(key)) + { + digestHistory.Add(key); + } + } + + private static string BuildQuarantineKey(RancherHubEventRecord record) + { + if (!string.IsNullOrWhiteSpace(record.Id)) + { + return $"quarantine:{record.Id}"; + } + + Span hash = stackalloc byte[32]; + var bytes = Encoding.UTF8.GetBytes(record.RawJson); + if (!SHA256.TryHashData(bytes, hash, out _)) + { + using var sha = SHA256.Create(); + hash = sha.ComputeHash(bytes); + } + + return $"quarantine:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static string NormalizeDigest(string digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return digest; + } + + var trimmed = digest.Trim(); + return trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) + ? trimmed.ToLowerInvariant() + : $"sha256:{trimmed.ToLowerInvariant()}"; + } + + private static VexDocumentFormat ResolveFormat(string? format) + { + if (string.IsNullOrWhiteSpace(format)) + { + return VexDocumentFormat.Csaf; + } + + return format.ToLowerInvariant() switch + { + "csaf" or "csaf_json" or "json" => VexDocumentFormat.Csaf, + "cyclonedx" or "cyclonedx_vex" => VexDocumentFormat.CycloneDx, + "openvex" => VexDocumentFormat.OpenVex, + "oci" or "oci_attestation" or "attestation" => VexDocumentFormat.OciAttestation, + _ => VexDocumentFormat.Csaf, + }; + } + + private sealed record EventProcessingResult(VexRawDocument? ProcessedDocument, bool Quarantined, DateTimeOffset? PublishedAt) + { + public static EventProcessingResult QuarantinedOnly { get; } = new(null, true, null); + + public static EventProcessingResult Skipped { get; } = new(null, false, null); + } +} diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/State/RancherHubCheckpointManager.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/State/RancherHubCheckpointManager.cs index 73c22674..9b443d0b 100644 --- a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/State/RancherHubCheckpointManager.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/State/RancherHubCheckpointManager.cs @@ -6,15 +6,15 @@ using System.Threading.Tasks; using StellaOps.Excititor.Core; using StellaOps.Excititor.Storage.Mongo; -namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State; - -internal sealed record RancherHubCheckpointState( - string? Cursor, - DateTimeOffset? LastPublishedAt, - DateTimeOffset? EffectiveSince, - ImmutableArray Digests); - -internal sealed class RancherHubCheckpointManager +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State; + +public sealed record RancherHubCheckpointState( + string? Cursor, + DateTimeOffset? LastPublishedAt, + DateTimeOffset? EffectiveSince, + ImmutableArray Digests); + +public sealed class RancherHubCheckpointManager { private const string CheckpointPrefix = "checkpoint:"; private readonly IVexConnectorStateRepository _repository; diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs index 84db1062..277b6417 100644 --- a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs @@ -1,309 +1,310 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Security.Cryptography; -using System.Text; -using System.Threading; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Connectors.Ubuntu.CSAF; -using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; -using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Storage.Mongo; -using System.IO.Abstractions.TestingHelpers; -using Xunit; - -namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors; - -public sealed class UbuntuCsafConnectorTests -{ - [Fact] - public async Task FetchAsync_IngestsNewDocument_UpdatesStateAndUsesEtag() - { - var baseUri = new Uri("https://ubuntu.test/security/csaf/"); - var indexUri = new Uri(baseUri, "index.json"); - var catalogUri = new Uri(baseUri, "stable/catalog.json"); - var advisoryUri = new Uri(baseUri, "stable/USN-2025-0001.json"); - - var manifest = CreateTestManifest(advisoryUri, "USN-2025-0001", "2025-10-18T00:00:00Z"); - var documentPayload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); - var documentSha = ComputeSha256(documentPayload); - - var indexJson = manifest.IndexJson; - var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", documentSha, StringComparison.Ordinal); - var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, documentPayload, expectedEtag: "etag-123"); - - var httpClient = new HttpClient(handler); - var httpFactory = new SingleClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger.Instance, TimeProvider.System); - - var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem); - var stateRepository = new InMemoryConnectorStateRepository(); - var connector = new UbuntuCsafConnector( - loader, - httpFactory, - stateRepository, - new[] { optionsValidator }, - NullLogger.Instance, - TimeProvider.System); - - var settings = new VexConnectorSettings(ImmutableDictionary.Empty); - await connector.ValidateAsync(settings, CancellationToken.None); - - var sink = new InMemoryRawSink(); - var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(doc); - } - - documents.Should().HaveCount(1); - sink.Documents.Should().HaveCount(1); - var stored = sink.Documents.Single(); - stored.Digest.Should().Be($"sha256:{documentSha}"); - stored.Metadata.TryGetValue("ubuntu.etag", out var storedEtag).Should().BeTrue(); - storedEtag.Should().Be("etag-123"); - - stateRepository.CurrentState.Should().NotBeNull(); - stateRepository.CurrentState!.DocumentDigests.Should().Contain($"sha256:{documentSha}"); - stateRepository.CurrentState.DocumentDigests.Should().Contain($"etag:{advisoryUri}|etag-123"); - stateRepository.CurrentState.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-18T00:00:00Z")); - - handler.DocumentRequestCount.Should().Be(1); - - // Second run: Expect connector to send If-None-Match and skip download via 304. - sink.Documents.Clear(); - documents.Clear(); - - await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(doc); - } - - documents.Should().BeEmpty(); - sink.Documents.Should().BeEmpty(); - handler.DocumentRequestCount.Should().Be(2); - handler.SeenIfNoneMatch.Should().Contain("\"etag-123\""); - } - - [Fact] - public async Task FetchAsync_SkipsWhenChecksumMismatch() - { - var baseUri = new Uri("https://ubuntu.test/security/csaf/"); - var indexUri = new Uri(baseUri, "index.json"); - var catalogUri = new Uri(baseUri, "stable/catalog.json"); - var advisoryUri = new Uri(baseUri, "stable/USN-2025-0002.json"); - - var manifest = CreateTestManifest(advisoryUri, "USN-2025-0002", "2025-10-18T00:00:00Z"); - var indexJson = manifest.IndexJson; - var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", new string('a', 64), StringComparison.Ordinal); - var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"), expectedEtag: "etag-999"); - - var httpClient = new HttpClient(handler); - var httpFactory = new SingleClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger.Instance, TimeProvider.System); - var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem); - var stateRepository = new InMemoryConnectorStateRepository(); - - var connector = new UbuntuCsafConnector( - loader, - httpFactory, - stateRepository, - new[] { optionsValidator }, - NullLogger.Instance, - TimeProvider.System); - - await connector.ValidateAsync(new VexConnectorSettings(ImmutableDictionary.Empty), CancellationToken.None); - - var sink = new InMemoryRawSink(); - var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(doc); - } - - documents.Should().BeEmpty(); - sink.Documents.Should().BeEmpty(); - stateRepository.CurrentState.Should().NotBeNull(); - stateRepository.CurrentState!.DocumentDigests.Should().BeEmpty(); - handler.DocumentRequestCount.Should().Be(1); - } - - private static (string IndexJson, string CatalogJson) CreateTestManifest(Uri advisoryUri, string advisoryId, string timestamp) - { - var indexJson = $$""" - { - "generated": "2025-10-18T00:00:00Z", - "channels": [ - { - "name": "stable", - "catalogUrl": "{{advisoryUri.GetLeftPart(UriPartial.Authority)}}/security/csaf/stable/catalog.json", - "sha256": "ignore" - } - ] - } - """; - - var catalogJson = $$""" - { - "resources": [ - { - "id": "{{advisoryId}}", - "type": "csaf", - "url": "{{advisoryUri}}", - "last_modified": "{{timestamp}}", - "hashes": { - "sha256": "{{SHA256}}" - }, - "etag": "\"etag-123\"", - "title": "{{advisoryId}}" - } - ] - } - """; - - return (indexJson, catalogJson); - } - - private static string ComputeSha256(ReadOnlySpan payload) - { - Span buffer = stackalloc byte[32]; - SHA256.HashData(payload, buffer); - return Convert.ToHexString(buffer).ToLowerInvariant(); - } - - private sealed class SingleClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class UbuntuTestHttpHandler : HttpMessageHandler - { - private readonly Uri _indexUri; - private readonly string _indexPayload; - private readonly Uri _catalogUri; - private readonly string _catalogPayload; - private readonly Uri _documentUri; - private readonly byte[] _documentPayload; - private readonly string _expectedEtag; - - public int DocumentRequestCount { get; private set; } - public List SeenIfNoneMatch { get; } = new(); - - public UbuntuTestHttpHandler(Uri indexUri, string indexPayload, Uri catalogUri, string catalogPayload, Uri documentUri, byte[] documentPayload, string expectedEtag) - { - _indexUri = indexUri; - _indexPayload = indexPayload; - _catalogUri = catalogUri; - _catalogPayload = catalogPayload; - _documentUri = documentUri; - _documentPayload = documentPayload; - _expectedEtag = expectedEtag; - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - if (request.RequestUri == _indexUri) - { - return Task.FromResult(CreateJsonResponse(_indexPayload)); - } - - if (request.RequestUri == _catalogUri) - { - return Task.FromResult(CreateJsonResponse(_catalogPayload)); - } - - if (request.RequestUri == _documentUri) - { - DocumentRequestCount++; - if (request.Headers.IfNoneMatch is { Count: > 0 }) - { - var header = request.Headers.IfNoneMatch.First().ToString(); - SeenIfNoneMatch.Add(header); - if (header.Trim('"') == _expectedEtag || header == $"\"{_expectedEtag}\"") - { - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified)); - } - } - - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new ByteArrayContent(_documentPayload), - }; - response.Headers.ETag = new EntityTagHeaderValue($"\"{_expectedEtag}\""); - response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); - return Task.FromResult(response); - } - - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) - { - Content = new StringContent($"No response configured for {request.RequestUri}"), - }); - } - - private static HttpResponseMessage CreateJsonResponse(string payload) - => new(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - }; - } - - private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository - { - public VexConnectorState? CurrentState { get; private set; } - - public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) - => ValueTask.FromResult(CurrentState); - - public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) - { - CurrentState = state; - return ValueTask.CompletedTask; - } - } - - private sealed class InMemoryRawSink : IVexRawDocumentSink - { - public List Documents { get; } = new(); - - public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) - { - Documents.Add(document); - return ValueTask.CompletedTask; - } - } - - private sealed class NoopSignatureVerifier : IVexSignatureVerifier - { - public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - } - - private sealed class NoopNormalizerRouter : IVexNormalizerRouter - { - public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); - } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors; + +public sealed class UbuntuCsafConnectorTests +{ + [Fact] + public async Task FetchAsync_IngestsNewDocument_UpdatesStateAndUsesEtag() + { + var baseUri = new Uri("https://ubuntu.test/security/csaf/"); + var indexUri = new Uri(baseUri, "index.json"); + var catalogUri = new Uri(baseUri, "stable/catalog.json"); + var advisoryUri = new Uri(baseUri, "stable/USN-2025-0001.json"); + + var manifest = CreateTestManifest(advisoryUri, "USN-2025-0001", "2025-10-18T00:00:00Z"); + var documentPayload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); + var documentSha = ComputeSha256(documentPayload); + + var indexJson = manifest.IndexJson; + var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", documentSha, StringComparison.Ordinal); + var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, documentPayload, expectedEtag: "etag-123"); + + var httpClient = new HttpClient(handler); + var httpFactory = new SingleClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger.Instance, TimeProvider.System); + + var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem); + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new UbuntuCsafConnector( + loader, + httpFactory, + stateRepository, + new[] { optionsValidator }, + NullLogger.Instance, + TimeProvider.System); + + var settings = new VexConnectorSettings(ImmutableDictionary.Empty); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + var stored = sink.Documents.Single(); + stored.Digest.Should().Be($"sha256:{documentSha}"); + stored.Metadata.TryGetValue("ubuntu.etag", out var storedEtag).Should().BeTrue(); + storedEtag.Should().Be("etag-123"); + + stateRepository.CurrentState.Should().NotBeNull(); + stateRepository.CurrentState!.DocumentDigests.Should().Contain($"sha256:{documentSha}"); + stateRepository.CurrentState.DocumentDigests.Should().Contain($"etag:{advisoryUri}|etag-123"); + stateRepository.CurrentState.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-18T00:00:00Z")); + + handler.DocumentRequestCount.Should().Be(1); + + // Second run: Expect connector to send If-None-Match and skip download via 304. + sink.Documents.Clear(); + documents.Clear(); + + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + handler.DocumentRequestCount.Should().Be(2); + handler.SeenIfNoneMatch.Should().Contain("\"etag-123\""); + } + + [Fact] + public async Task FetchAsync_SkipsWhenChecksumMismatch() + { + var baseUri = new Uri("https://ubuntu.test/security/csaf/"); + var indexUri = new Uri(baseUri, "index.json"); + var catalogUri = new Uri(baseUri, "stable/catalog.json"); + var advisoryUri = new Uri(baseUri, "stable/USN-2025-0002.json"); + + var manifest = CreateTestManifest(advisoryUri, "USN-2025-0002", "2025-10-18T00:00:00Z"); + var indexJson = manifest.IndexJson; + var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", new string('a', 64), StringComparison.Ordinal); + var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"), expectedEtag: "etag-999"); + + var httpClient = new HttpClient(handler); + var httpFactory = new SingleClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger.Instance, TimeProvider.System); + var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem); + var stateRepository = new InMemoryConnectorStateRepository(); + + var connector = new UbuntuCsafConnector( + loader, + httpFactory, + stateRepository, + new[] { optionsValidator }, + NullLogger.Instance, + TimeProvider.System); + + await connector.ValidateAsync(new VexConnectorSettings(ImmutableDictionary.Empty), CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + stateRepository.CurrentState.Should().NotBeNull(); + stateRepository.CurrentState!.DocumentDigests.Should().BeEmpty(); + handler.DocumentRequestCount.Should().Be(1); + } + + private static (string IndexJson, string CatalogJson) CreateTestManifest(Uri advisoryUri, string advisoryId, string timestamp) + { + var indexJson = """ + { + "generated": "2025-10-18T00:00:00Z", + "channels": [ + { + "name": "stable", + "catalogUrl": "{{advisoryUri.GetLeftPart(UriPartial.Authority)}}/security/csaf/stable/catalog.json", + "sha256": "ignore" + } + ] + } + """; + + var catalogJson = """ + { + "resources": [ + { + "id": "{{advisoryId}}", + "type": "csaf", + "url": "{{advisoryUri}}", + "last_modified": "{{timestamp}}", + "hashes": { + "sha256": "{{SHA256}}" + }, + "etag": "\"etag-123\"", + "title": "{{advisoryId}}" + } + ] + } + """; + + return (indexJson, catalogJson); + } + + private static string ComputeSha256(ReadOnlySpan payload) + { + Span buffer = stackalloc byte[32]; + SHA256.HashData(payload, buffer); + return Convert.ToHexString(buffer).ToLowerInvariant(); + } + + private sealed class SingleClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class UbuntuTestHttpHandler : HttpMessageHandler + { + private readonly Uri _indexUri; + private readonly string _indexPayload; + private readonly Uri _catalogUri; + private readonly string _catalogPayload; + private readonly Uri _documentUri; + private readonly byte[] _documentPayload; + private readonly string _expectedEtag; + + public int DocumentRequestCount { get; private set; } + public List SeenIfNoneMatch { get; } = new(); + + public UbuntuTestHttpHandler(Uri indexUri, string indexPayload, Uri catalogUri, string catalogPayload, Uri documentUri, byte[] documentPayload, string expectedEtag) + { + _indexUri = indexUri; + _indexPayload = indexPayload; + _catalogUri = catalogUri; + _catalogPayload = catalogPayload; + _documentUri = documentUri; + _documentPayload = documentPayload; + _expectedEtag = expectedEtag; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri == _indexUri) + { + return Task.FromResult(CreateJsonResponse(_indexPayload)); + } + + if (request.RequestUri == _catalogUri) + { + return Task.FromResult(CreateJsonResponse(_catalogPayload)); + } + + if (request.RequestUri == _documentUri) + { + DocumentRequestCount++; + if (request.Headers.IfNoneMatch is { Count: > 0 }) + { + var header = request.Headers.IfNoneMatch.First().ToString(); + SeenIfNoneMatch.Add(header); + if (header.Trim('"') == _expectedEtag || header == $"\"{_expectedEtag}\"") + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified)); + } + } + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(_documentPayload), + }; + response.Headers.ETag = new EntityTagHeaderValue($"\"{_expectedEtag}\""); + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + return Task.FromResult(response); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"No response configured for {request.RequestUri}"), + }); + } + + private static HttpResponseMessage CreateJsonResponse(string payload) + => new(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? CurrentState { get; private set; } + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(CurrentState); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + CurrentState = state; + return ValueTask.CompletedTask; + } + } + + private sealed class InMemoryRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } +} diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs index 9592ba75..56d323ef 100644 --- a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs @@ -1,502 +1,516 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Globalization; -using System.Net; -using System.Net.Http; -using System.Runtime.CompilerServices; -using System.Security.Cryptography; -using System.Text.Json; -using Microsoft.Extensions.Logging; -using StellaOps.Excititor.Connectors.Abstractions; -using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; -using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Storage.Mongo; - -namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF; - -public sealed class UbuntuCsafConnector : VexConnectorBase -{ - private const string EtagTokenPrefix = "etag:"; - - private static readonly VexConnectorDescriptor DescriptorInstance = new( - id: "excititor:ubuntu", - kind: VexProviderKind.Distro, - displayName: "Ubuntu CSAF") - { - Tags = ImmutableArray.Create("ubuntu", "csaf", "usn"), - }; - - private readonly UbuntuCatalogLoader _catalogLoader; - private readonly IHttpClientFactory _httpClientFactory; - private readonly IVexConnectorStateRepository _stateRepository; - private readonly IEnumerable> _validators; - - private UbuntuConnectorOptions? _options; - private UbuntuCatalogResult? _catalog; - - public UbuntuCsafConnector( - UbuntuCatalogLoader catalogLoader, - IHttpClientFactory httpClientFactory, - IVexConnectorStateRepository stateRepository, - IEnumerable> validators, - ILogger logger, - TimeProvider timeProvider) - : base(DescriptorInstance, logger, timeProvider) - { - _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader)); - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _validators = validators ?? Array.Empty>(); - } - - public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) - { - _options = VexConnectorOptionsBinder.Bind( - Descriptor, - settings, - validators: _validators); - - _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - LogConnectorEvent(LogLevel.Information, "validate", "Ubuntu CSAF index loaded.", new Dictionary - { - ["channelCount"] = _catalog.Metadata.Channels.Length, - ["fromOffline"] = _catalog.FromOfflineSnapshot, - }); - } - - public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - if (_options is null) - { - throw new InvalidOperationException("Connector must be validated before fetch operations."); - } - - if (_catalog is null) - { - _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - } - - var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); - var knownTokens = state?.DocumentDigests ?? ImmutableArray.Empty; - var digestSet = new HashSet(StringComparer.OrdinalIgnoreCase); - var tokenSet = new HashSet(StringComparer.Ordinal); - var tokenList = new List(knownTokens.Length + 16); - var etagMap = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var token in knownTokens) - { - tokenSet.Add(token); - tokenList.Add(token); - if (TryParseEtagToken(token, out var uri, out var etag)) - { - etagMap[uri] = etag; - } - else - { - digestSet.Add(token); - } - } - - var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; - var latestTimestamp = state?.LastUpdated ?? since; - var stateChanged = false; - - foreach (var channel in _catalog.Metadata.Channels) - { - await foreach (var entry in EnumerateChannelResourcesAsync(channel, cancellationToken).ConfigureAwait(false)) - { - var entryTimestamp = entry.LastModified ?? channel.LastUpdated ?? _catalog.Metadata.GeneratedAt; - if (entryTimestamp <= since) - { - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - continue; - } - - var expectedDigest = entry.Sha256 is null ? null : NormalizeDigest(entry.Sha256); - if (expectedDigest is not null && digestSet.Contains(expectedDigest)) - { - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - continue; - } - - etagMap.TryGetValue(entry.DocumentUri.ToString(), out var knownEtag); - - var download = await DownloadDocumentAsync(entry, knownEtag, cancellationToken).ConfigureAwait(false); - if (download is null) - { - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - continue; - } - - var document = download.Document; - if (!digestSet.Add(document.Digest)) - { - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - continue; - } - - await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); - if (tokenSet.Add(document.Digest)) - { - tokenList.Add(document.Digest); - } - - if (!string.IsNullOrWhiteSpace(download.ETag)) - { - var etagValue = download.ETag!; - etagMap[entry.DocumentUri.ToString()] = etagValue; - var etagToken = CreateEtagToken(entry.DocumentUri, etagValue); - if (tokenSet.Add(etagToken)) - { - tokenList.Add(etagToken); - } - } - - stateChanged = true; - if (entryTimestamp > latestTimestamp) - { - latestTimestamp = entryTimestamp; - } - - yield return document; - } - } - - if (stateChanged || latestTimestamp > (state?.LastUpdated ?? DateTimeOffset.MinValue)) - { - var newState = new VexConnectorState( - Descriptor.Id, - latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, - tokenList.ToImmutableArray()); - - await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); - } - } - - public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => throw new NotSupportedException("UbuntuCsafConnector relies on CSAF normalizers for document processing."); - - public UbuntuCatalogResult? GetCachedCatalog() => _catalog; - - private async IAsyncEnumerable EnumerateChannelResourcesAsync(UbuChannelCatalog channel, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); - HttpResponseMessage? response = null; - try - { - response = await client.GetAsync(channel.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); - - if (!document.RootElement.TryGetProperty("resources", out var resourcesElement) || resourcesElement.ValueKind != JsonValueKind.Array) - { - LogConnectorEvent(LogLevel.Warning, "fetch.catalog.empty", "Ubuntu CSAF channel catalog missing 'resources' array.", new Dictionary - { - ["channel"] = channel.Name, - ["catalog"] = channel.CatalogUri.ToString(), - }); - yield break; - } - - foreach (var resource in resourcesElement.EnumerateArray()) - { - var type = GetString(resource, "type"); - if (type is not null && !type.Equals("csaf", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var uriText = GetString(resource, "url") - ?? GetString(resource, "canonical") - ?? GetString(resource, "download") - ?? GetString(resource, "uri"); - - if (uriText is null || !Uri.TryCreate(uriText, UriKind.Absolute, out var documentUri)) - { - continue; - } - - var sha256 = TryGetHash(resource); - var etag = GetString(resource, "etag"); - var lastModified = ParseDate(resource, "last_modified") - ?? ParseDate(resource, "published") - ?? ParseDate(resource, "released") - ?? channel.LastUpdated; - var title = GetString(resource, "title"); - var version = GetString(resource, "version"); - var advisoryId = GetString(resource, "id") ?? ExtractAdvisoryId(documentUri, title); - - yield return new UbuntuCatalogEntry( - channel.Name, - advisoryId, - documentUri, - sha256, - etag, - lastModified, - title, - version); - } - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - LogConnectorEvent(LogLevel.Warning, "fetch.catalog.failure", "Failed to enumerate Ubuntu CSAF channel catalog.", new Dictionary - { - ["channel"] = channel.Name, - ["catalog"] = channel.CatalogUri.ToString(), - }, ex); - } - finally - { - response?.Dispose(); - } - } - - private async Task DownloadDocumentAsync(UbuntuCatalogEntry entry, string? knownEtag, CancellationToken cancellationToken) - { - var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); - using var request = new HttpRequestMessage(HttpMethod.Get, entry.DocumentUri); - if (!string.IsNullOrWhiteSpace(knownEtag)) - { - request.Headers.IfNoneMatch.TryParseAdd(EnsureQuoted(knownEtag)); - } - - HttpResponseMessage? response = null; - try - { - response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - - if (response.StatusCode == HttpStatusCode.NotModified) - { - LogConnectorEvent(LogLevel.Debug, "fetch.document.not_modified", "Ubuntu CSAF document not modified per ETag.", new Dictionary - { - ["uri"] = entry.DocumentUri.ToString(), - ["etag"] = knownEtag, - }); - return null; - } - - response.EnsureSuccessStatusCode(); - - var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); - if (entry.Sha256 is not null) - { - var expected = NormalizeDigest(entry.Sha256); - var actual = "sha256:" + ComputeSha256Hex(payload); - if (!string.Equals(expected, actual, StringComparison.OrdinalIgnoreCase)) - { - LogConnectorEvent(LogLevel.Warning, "fetch.document.checksum_mismatch", "Ubuntu CSAF document checksum mismatch; skipping document.", new Dictionary - { - ["uri"] = entry.DocumentUri.ToString(), - ["expected"] = expected, - ["actual"] = actual, - }); - return null; - } - } - - var etagHeader = response.Headers.ETag?.Tag; - var etagValue = !string.IsNullOrWhiteSpace(etagHeader) - ? Unquote(etagHeader!) - : entry.ETag is null ? null : Unquote(entry.ETag); - - var metadata = BuildMetadata(builder => - { - builder.Add("ubuntu.channel", entry.Channel); - builder.Add("ubuntu.uri", entry.DocumentUri.ToString()); - if (!string.IsNullOrWhiteSpace(entry.AdvisoryId)) - { - builder.Add("ubuntu.advisoryId", entry.AdvisoryId); - } - - if (!string.IsNullOrWhiteSpace(entry.Title)) - { - builder.Add("ubuntu.title", entry.Title!); - } - - if (!string.IsNullOrWhiteSpace(entry.Version)) - { - builder.Add("ubuntu.version", entry.Version!); - } - - if (entry.LastModified is { } modified) - { - builder.Add("ubuntu.lastModified", modified.ToString("O")); - } - - if (entry.Sha256 is not null) - { - builder.Add("ubuntu.sha256", NormalizeDigest(entry.Sha256)); - } - - if (!string.IsNullOrWhiteSpace(etagValue)) - { - builder.Add("ubuntu.etag", etagValue!); - } - }); - - var document = CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload, metadata); - return new DownloadResult(document, etagValue); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - LogConnectorEvent(LogLevel.Warning, "fetch.document.failure", "Failed to download Ubuntu CSAF document.", new Dictionary - { - ["uri"] = entry.DocumentUri.ToString(), - }, ex); - return null; - } - finally - { - response?.Dispose(); - } - } - - private static string NormalizeDigest(string value) - { - var trimmed = value.Trim(); - if (trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) - { - trimmed = trimmed[7..]; - } - - return "sha256:" + trimmed.Replace(" ", string.Empty, StringComparison.Ordinal).ToLowerInvariant(); - } - - private static string ComputeSha256Hex(ReadOnlySpan payload) - { - Span buffer = stackalloc byte[32]; - SHA256.HashData(payload, buffer); - return Convert.ToHexString(buffer).ToLowerInvariant(); - } - - private static string? GetString(JsonElement element, string propertyName) - { - if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String) - { - var value = property.GetString(); - return string.IsNullOrWhiteSpace(value) ? null : value; - } - - return null; - } - - private static string? TryGetHash(JsonElement resource) - { - if (resource.TryGetProperty("hashes", out var hashesElement) && hashesElement.ValueKind == JsonValueKind.Object) - { - if (hashesElement.TryGetProperty("sha256", out var hash) && hash.ValueKind == JsonValueKind.String) - { - var value = hash.GetString(); - if (!string.IsNullOrWhiteSpace(value)) - { - return value; - } - } - } - - return GetString(resource, "sha256"); - } - - private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) - { - var text = GetString(element, propertyName); - if (text is null) - { - return null; - } - - return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var value) - ? value - : (DateTimeOffset?)null; - } - - private static string ExtractAdvisoryId(Uri uri, string? title) - { - if (!string.IsNullOrWhiteSpace(title)) - { - return title!; - } - - var segments = uri.Segments; - if (segments.Length > 0) - { - var candidate = segments[^1].Trim('/'); - if (candidate.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) - { - candidate = candidate[..^5]; - } - - if (!string.IsNullOrWhiteSpace(candidate)) - { - return candidate; - } - } - - return uri.AbsolutePath.Trim('/'); - } - - private static string EnsureQuoted(string value) - { - var trimmed = value.Trim(); - return trimmed.StartsWith('"') ? trimmed : $"\"{trimmed}\""; - } - - private static string Unquote(string value) - => value.Trim().Trim('"'); - - private static string CreateEtagToken(Uri uri, string etag) - => $"{EtagTokenPrefix}{uri}|{etag}"; - - private static bool TryParseEtagToken(string token, out string uri, out string etag) - { - uri = string.Empty; - etag = string.Empty; - if (!token.StartsWith(EtagTokenPrefix, StringComparison.Ordinal)) - { - return false; - } - - var separatorIndex = token.IndexOf('|', EtagTokenPrefix.Length); - if (separatorIndex < 0 || separatorIndex == EtagTokenPrefix.Length) - { - return false; - } - - uri = token[EtagTokenPrefix.Length..separatorIndex]; - etag = token[(separatorIndex + 1)..]; - return !string.IsNullOrWhiteSpace(uri) && !string.IsNullOrWhiteSpace(etag); - } - - private sealed record UbuntuCatalogEntry( - string Channel, - string? AdvisoryId, - Uri DocumentUri, - string? Sha256, - string? ETag, - DateTimeOffset? LastModified, - string? Title, - string? Version); - - private sealed record DownloadResult(VexRawDocument Document, string? ETag); -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF; + +public sealed class UbuntuCsafConnector : VexConnectorBase +{ + private const string EtagTokenPrefix = "etag:"; + + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "excititor:ubuntu", + kind: VexProviderKind.Distro, + displayName: "Ubuntu CSAF") + { + Tags = ImmutableArray.Create("ubuntu", "csaf", "usn"), + }; + + private readonly UbuntuCatalogLoader _catalogLoader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IVexConnectorStateRepository _stateRepository; + private readonly IEnumerable> _validators; + + private UbuntuConnectorOptions? _options; + private UbuntuCatalogResult? _catalog; + + public UbuntuCsafConnector( + UbuntuCatalogLoader catalogLoader, + IHttpClientFactory httpClientFactory, + IVexConnectorStateRepository stateRepository, + IEnumerable> validators, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + LogConnectorEvent(LogLevel.Information, "validate", "Ubuntu CSAF index loaded.", new Dictionary + { + ["channelCount"] = _catalog.Metadata.Channels.Length, + ["fromOffline"] = _catalog.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_catalog is null) + { + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + } + + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + var knownTokens = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestSet = new HashSet(StringComparer.OrdinalIgnoreCase); + var tokenSet = new HashSet(StringComparer.Ordinal); + var tokenList = new List(knownTokens.Length + 16); + var etagMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var token in knownTokens) + { + tokenSet.Add(token); + tokenList.Add(token); + if (TryParseEtagToken(token, out var uri, out var etag)) + { + etagMap[uri] = etag; + } + else + { + digestSet.Add(token); + } + } + + var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; + var latestTimestamp = state?.LastUpdated ?? since; + var stateChanged = false; + + foreach (var channel in _catalog.Metadata.Channels) + { + await foreach (var entry in EnumerateChannelResourcesAsync(channel, cancellationToken).ConfigureAwait(false)) + { + var entryTimestamp = entry.LastModified ?? channel.LastUpdated ?? _catalog.Metadata.GeneratedAt; + if (entryTimestamp <= since) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + var expectedDigest = entry.Sha256 is null ? null : NormalizeDigest(entry.Sha256); + if (expectedDigest is not null && digestSet.Contains(expectedDigest)) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + etagMap.TryGetValue(entry.DocumentUri.ToString(), out var knownEtag); + + var download = await DownloadDocumentAsync(entry, knownEtag, cancellationToken).ConfigureAwait(false); + if (download is null) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + var document = download.Document; + if (!digestSet.Add(document.Digest)) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); + if (tokenSet.Add(document.Digest)) + { + tokenList.Add(document.Digest); + } + + if (!string.IsNullOrWhiteSpace(download.ETag)) + { + var etagValue = download.ETag!; + etagMap[entry.DocumentUri.ToString()] = etagValue; + var etagToken = CreateEtagToken(entry.DocumentUri, etagValue); + if (tokenSet.Add(etagToken)) + { + tokenList.Add(etagToken); + } + } + + stateChanged = true; + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + yield return document; + } + } + + if (stateChanged || latestTimestamp > (state?.LastUpdated ?? DateTimeOffset.MinValue)) + { + var newState = new VexConnectorState( + Descriptor.Id, + latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, + tokenList.ToImmutableArray()); + + await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); + } + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("UbuntuCsafConnector relies on CSAF normalizers for document processing."); + + public UbuntuCatalogResult? GetCachedCatalog() => _catalog; + + private async IAsyncEnumerable EnumerateChannelResourcesAsync(UbuChannelCatalog channel, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); + HttpResponseMessage? response = null; + List? entries = null; + + try + { + response = await client.GetAsync(channel.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + + if (!document.RootElement.TryGetProperty("resources", out var resourcesElement) || resourcesElement.ValueKind != JsonValueKind.Array) + { + LogConnectorEvent(LogLevel.Warning, "fetch.catalog.empty", "Ubuntu CSAF channel catalog missing 'resources' array.", new Dictionary + { + ["channel"] = channel.Name, + ["catalog"] = channel.CatalogUri.ToString(), + }); + yield break; + } + + entries = new List(resourcesElement.GetArrayLength()); + foreach (var resource in resourcesElement.EnumerateArray()) + { + var type = GetString(resource, "type"); + if (type is not null && !type.Equals("csaf", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var uriText = GetString(resource, "url") + ?? GetString(resource, "canonical") + ?? GetString(resource, "download") + ?? GetString(resource, "uri"); + + if (uriText is null || !Uri.TryCreate(uriText, UriKind.Absolute, out var documentUri)) + { + continue; + } + + var sha256 = TryGetHash(resource); + var etag = GetString(resource, "etag"); + var lastModified = ParseDate(resource, "last_modified") + ?? ParseDate(resource, "published") + ?? ParseDate(resource, "released") + ?? channel.LastUpdated; + var title = GetString(resource, "title"); + var version = GetString(resource, "version"); + var advisoryId = GetString(resource, "id") ?? ExtractAdvisoryId(documentUri, title); + + entries.Add(new UbuntuCatalogEntry( + channel.Name, + advisoryId, + documentUri, + sha256, + etag, + lastModified, + title, + version)); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectorEvent(LogLevel.Warning, "fetch.catalog.failure", "Failed to enumerate Ubuntu CSAF channel catalog.", new Dictionary + { + ["channel"] = channel.Name, + ["catalog"] = channel.CatalogUri.ToString(), + }, ex); + } + finally + { + response?.Dispose(); + } + + if (entries is null) + { + yield break; + } + + foreach (var entry in entries) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return entry; + } + } + + private async Task DownloadDocumentAsync(UbuntuCatalogEntry entry, string? knownEtag, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, entry.DocumentUri); + if (!string.IsNullOrWhiteSpace(knownEtag)) + { + request.Headers.IfNoneMatch.TryParseAdd(EnsureQuoted(knownEtag)); + } + + HttpResponseMessage? response = null; + try + { + response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + LogConnectorEvent(LogLevel.Debug, "fetch.document.not_modified", "Ubuntu CSAF document not modified per ETag.", new Dictionary + { + ["uri"] = entry.DocumentUri.ToString(), + ["etag"] = knownEtag, + }); + return null; + } + + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + if (entry.Sha256 is not null) + { + var expected = NormalizeDigest(entry.Sha256); + var actual = "sha256:" + ComputeSha256Hex(payload); + if (!string.Equals(expected, actual, StringComparison.OrdinalIgnoreCase)) + { + LogConnectorEvent(LogLevel.Warning, "fetch.document.checksum_mismatch", "Ubuntu CSAF document checksum mismatch; skipping document.", new Dictionary + { + ["uri"] = entry.DocumentUri.ToString(), + ["expected"] = expected, + ["actual"] = actual, + }); + return null; + } + } + + var etagHeader = response.Headers.ETag?.Tag; + var etagValue = !string.IsNullOrWhiteSpace(etagHeader) + ? Unquote(etagHeader!) + : entry.ETag is null ? null : Unquote(entry.ETag); + + var metadata = BuildMetadata(builder => + { + builder.Add("ubuntu.channel", entry.Channel); + builder.Add("ubuntu.uri", entry.DocumentUri.ToString()); + if (!string.IsNullOrWhiteSpace(entry.AdvisoryId)) + { + builder.Add("ubuntu.advisoryId", entry.AdvisoryId); + } + + if (!string.IsNullOrWhiteSpace(entry.Title)) + { + builder.Add("ubuntu.title", entry.Title!); + } + + if (!string.IsNullOrWhiteSpace(entry.Version)) + { + builder.Add("ubuntu.version", entry.Version!); + } + + if (entry.LastModified is { } modified) + { + builder.Add("ubuntu.lastModified", modified.ToString("O")); + } + + if (entry.Sha256 is not null) + { + builder.Add("ubuntu.sha256", NormalizeDigest(entry.Sha256)); + } + + if (!string.IsNullOrWhiteSpace(etagValue)) + { + builder.Add("ubuntu.etag", etagValue!); + } + }); + + var document = CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload, metadata); + return new DownloadResult(document, etagValue); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectorEvent(LogLevel.Warning, "fetch.document.failure", "Failed to download Ubuntu CSAF document.", new Dictionary + { + ["uri"] = entry.DocumentUri.ToString(), + }, ex); + return null; + } + finally + { + response?.Dispose(); + } + } + + private static string NormalizeDigest(string value) + { + var trimmed = value.Trim(); + if (trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed[7..]; + } + + return "sha256:" + trimmed.Replace(" ", string.Empty, StringComparison.Ordinal).ToLowerInvariant(); + } + + private static string ComputeSha256Hex(ReadOnlySpan payload) + { + Span buffer = stackalloc byte[32]; + SHA256.HashData(payload, buffer); + return Convert.ToHexString(buffer).ToLowerInvariant(); + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String) + { + var value = property.GetString(); + return string.IsNullOrWhiteSpace(value) ? null : value; + } + + return null; + } + + private static string? TryGetHash(JsonElement resource) + { + if (resource.TryGetProperty("hashes", out var hashesElement) && hashesElement.ValueKind == JsonValueKind.Object) + { + if (hashesElement.TryGetProperty("sha256", out var hash) && hash.ValueKind == JsonValueKind.String) + { + var value = hash.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + } + + return GetString(resource, "sha256"); + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var text = GetString(element, propertyName); + if (text is null) + { + return null; + } + + return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var value) + ? value + : (DateTimeOffset?)null; + } + + private static string ExtractAdvisoryId(Uri uri, string? title) + { + if (!string.IsNullOrWhiteSpace(title)) + { + return title!; + } + + var segments = uri.Segments; + if (segments.Length > 0) + { + var candidate = segments[^1].Trim('/'); + if (candidate.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + candidate = candidate[..^5]; + } + + if (!string.IsNullOrWhiteSpace(candidate)) + { + return candidate; + } + } + + return uri.AbsolutePath.Trim('/'); + } + + private static string EnsureQuoted(string value) + { + var trimmed = value.Trim(); + return trimmed.StartsWith('"') ? trimmed : $"\"{trimmed}\""; + } + + private static string Unquote(string value) + => value.Trim().Trim('"'); + + private static string CreateEtagToken(Uri uri, string etag) + => $"{EtagTokenPrefix}{uri}|{etag}"; + + private static bool TryParseEtagToken(string token, out string uri, out string etag) + { + uri = string.Empty; + etag = string.Empty; + if (!token.StartsWith(EtagTokenPrefix, StringComparison.Ordinal)) + { + return false; + } + + var separatorIndex = token.IndexOf('|', EtagTokenPrefix.Length); + if (separatorIndex < 0 || separatorIndex == EtagTokenPrefix.Length) + { + return false; + } + + uri = token[EtagTokenPrefix.Length..separatorIndex]; + etag = token[(separatorIndex + 1)..]; + return !string.IsNullOrWhiteSpace(uri) && !string.IsNullOrWhiteSpace(etag); + } + + private sealed record UbuntuCatalogEntry( + string Channel, + string? AdvisoryId, + Uri DocumentUri, + string? Sha256, + string? ETag, + DateTimeOffset? LastModified, + string? Title, + string? Version); + + private sealed record DownloadResult(VexRawDocument Document, string? ETag); +} diff --git a/src/StellaOps.Excititor.Export.Tests/ExportEngineTests.cs b/src/StellaOps.Excititor.Export.Tests/ExportEngineTests.cs index 93ddadff..0d488a80 100644 --- a/src/StellaOps.Excititor.Export.Tests/ExportEngineTests.cs +++ b/src/StellaOps.Excititor.Export.Tests/ExportEngineTests.cs @@ -1,277 +1,277 @@ -using System; -using System.Collections.Immutable; -using System.IO; -using System.Text; -using Microsoft.Extensions.Logging.Abstractions; -using MongoDB.Driver; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Export; -using StellaOps.Excititor.Policy; -using StellaOps.Excititor.Storage.Mongo; -using Xunit; - -namespace StellaOps.Excititor.Export.Tests; - -public sealed class ExportEngineTests -{ - [Fact] - public async Task ExportAsync_GeneratesAndCachesManifest() - { - var store = new InMemoryExportStore(); - var evaluator = new StaticPolicyEvaluator("baseline/v1"); - var dataSource = new InMemoryExportDataSource(); - var exporter = new DummyExporter(VexExportFormat.Json); - var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger.Instance); - - var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); - var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow, ForceRefresh: false); - - var manifest = await engine.ExportAsync(context, CancellationToken.None); - - Assert.False(manifest.FromCache); - Assert.Equal(VexExportFormat.Json, manifest.Format); - Assert.Equal("baseline/v1", manifest.ConsensusRevision); - Assert.Equal(1, manifest.ClaimCount); - - // second call hits cache - var cached = await engine.ExportAsync(context, CancellationToken.None); - Assert.True(cached.FromCache); - Assert.Equal(manifest.ExportId, cached.ExportId); - } - - [Fact] - public async Task ExportAsync_ForceRefreshInvalidatesCacheEntry() - { - var store = new InMemoryExportStore(); - var evaluator = new StaticPolicyEvaluator("baseline/v1"); - var dataSource = new InMemoryExportDataSource(); - var exporter = new DummyExporter(VexExportFormat.Json); - var cacheIndex = new RecordingCacheIndex(); - var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger.Instance, cacheIndex); - - var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); - var initialContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); - _ = await engine.ExportAsync(initialContext, CancellationToken.None); - - var refreshContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow.AddMinutes(1), ForceRefresh: true); - var refreshed = await engine.ExportAsync(refreshContext, CancellationToken.None); - - Assert.False(refreshed.FromCache); - var signature = VexQuerySignature.FromQuery(refreshContext.Query); - Assert.True(cacheIndex.RemoveCalls.TryGetValue((signature.Value, refreshContext.Format), out var removed)); - Assert.True(removed); - } - - [Fact] - public async Task ExportAsync_WritesArtifactsToAllStores() - { - var store = new InMemoryExportStore(); - var evaluator = new StaticPolicyEvaluator("baseline/v1"); - var dataSource = new InMemoryExportDataSource(); - var exporter = new DummyExporter(VexExportFormat.Json); - var recorder1 = new RecordingArtifactStore(); - var recorder2 = new RecordingArtifactStore(); - var engine = new VexExportEngine( - store, - evaluator, - dataSource, - new[] { exporter }, - NullLogger.Instance, - cacheIndex: null, - artifactStores: new[] { recorder1, recorder2 }); - - var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); - var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); - - await engine.ExportAsync(context, CancellationToken.None); - - Assert.Equal(1, recorder1.SaveCount); - Assert.Equal(1, recorder2.SaveCount); - } - - [Fact] - public async Task ExportAsync_AttachesAttestationMetadata() - { - var store = new InMemoryExportStore(); - var evaluator = new StaticPolicyEvaluator("baseline/v1"); - var dataSource = new InMemoryExportDataSource(); - var exporter = new DummyExporter(VexExportFormat.Json); - var attestation = new RecordingAttestationClient(); - var engine = new VexExportEngine( - store, - evaluator, - dataSource, - new[] { exporter }, - NullLogger.Instance, - cacheIndex: null, - artifactStores: null, - attestationClient: attestation); - - var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); - var requestedAt = DateTimeOffset.UtcNow; - var context = new VexExportRequestContext(query, VexExportFormat.Json, requestedAt); - - var manifest = await engine.ExportAsync(context, CancellationToken.None); - - Assert.NotNull(attestation.LastRequest); - Assert.Equal(manifest.ExportId, attestation.LastRequest!.ExportId); - Assert.NotNull(manifest.Attestation); - Assert.Equal(attestation.Response.Attestation.EnvelopeDigest, manifest.Attestation!.EnvelopeDigest); - Assert.Equal(attestation.Response.Attestation.PredicateType, manifest.Attestation.PredicateType); - - Assert.NotNull(store.LastSavedManifest); - Assert.Equal(manifest.Attestation, store.LastSavedManifest!.Attestation); - } - - private sealed class InMemoryExportStore : IVexExportStore - { - private readonly Dictionary _store = new(StringComparer.Ordinal); - - public VexExportManifest? LastSavedManifest { get; private set; } - - public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) - { - var key = CreateKey(signature.Value, format); - _store.TryGetValue(key, out var manifest); - return ValueTask.FromResult(manifest); - } - - public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) - { - var key = CreateKey(manifest.QuerySignature.Value, manifest.Format); - _store[key] = manifest; - LastSavedManifest = manifest; - return ValueTask.CompletedTask; - } - - private static string CreateKey(string signature, VexExportFormat format) - => FormattableString.Invariant($"{signature}|{format}"); - } - - private sealed class RecordingAttestationClient : IVexAttestationClient - { - public VexAttestationRequest? LastRequest { get; private set; } - - public VexAttestationResponse Response { get; } = new VexAttestationResponse( - new VexAttestationMetadata( - predicateType: "https://stella-ops.org/attestations/vex-export", - rekor: new VexRekorReference("0.2", "rekor://entry", "123"), - envelopeDigest: "sha256:envelope", - signedAt: DateTimeOffset.UnixEpoch), - ImmutableDictionary.Empty); - - public ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) - { - LastRequest = request; - return ValueTask.FromResult(Response); - } - - public ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary.Empty)); - } - - private sealed class RecordingCacheIndex : IVexCacheIndex - { - public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new(); - - public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - - public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) - => ValueTask.CompletedTask; - - public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - RemoveCalls[(signature.Value, format)] = true; - return ValueTask.CompletedTask; - } - } - - private sealed class RecordingArtifactStore : IVexArtifactStore - { - public int SaveCount { get; private set; } - - public ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) - { - SaveCount++; - return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory", artifact.Content.Length, artifact.Metadata)); - } - - public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - => ValueTask.CompletedTask; - - public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - } - - private sealed class StaticPolicyEvaluator : IVexPolicyEvaluator - { - public StaticPolicyEvaluator(string version) - { - Version = version; - } - - public string Version { get; } - - public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; - - public double GetProviderWeight(VexProvider provider) => 1.0; - - public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) - { - rejectionReason = null; - return true; - } - } - - private sealed class InMemoryExportDataSource : IVexExportDataSource - { - public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) - { - var claim = new VexClaim( - "CVE-2025-0001", - "vendor", - new VexProduct("pkg:demo/app", "Demo"), - VexClaimStatus.Affected, - new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:demo", new Uri("https://example.org/demo")), - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow); - - var consensus = new VexConsensus( - "CVE-2025-0001", - claim.Product, - VexConsensusStatus.Affected, - DateTimeOffset.UtcNow, - new[] { new VexConsensusSource("vendor", VexClaimStatus.Affected, "sha256:demo", 1.0) }, - conflicts: null, - policyVersion: "baseline/v1", - summary: "affected"); - - return ValueTask.FromResult(new VexExportDataSet( - ImmutableArray.Create(consensus), - ImmutableArray.Create(claim), - ImmutableArray.Create("vendor"))); - } - } - - private sealed class DummyExporter : IVexExporter - { - public DummyExporter(VexExportFormat format) - { - Format = format; - } - - public VexExportFormat Format { get; } - - public VexContentAddress Digest(VexExportRequest request) - => new("sha256", "deadbeef"); - - public ValueTask SerializeAsync(VexExportRequest request, Stream output, CancellationToken cancellationToken) - { - var bytes = System.Text.Encoding.UTF8.GetBytes("{}"); - output.Write(bytes); - return ValueTask.FromResult(new VexExportResult(new VexContentAddress("sha256", "deadbeef"), bytes.Length, ImmutableDictionary.Empty)); - } - } - -} +using System; +using System.Collections.Immutable; +using System.IO; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; +using Xunit; + +namespace StellaOps.Excititor.Export.Tests; + +public sealed class ExportEngineTests +{ + [Fact] + public async Task ExportAsync_GeneratesAndCachesManifest() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger.Instance); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow, ForceRefresh: false); + + var manifest = await engine.ExportAsync(context, CancellationToken.None); + + Assert.False(manifest.FromCache); + Assert.Equal(VexExportFormat.Json, manifest.Format); + Assert.Equal("baseline/v1", manifest.ConsensusRevision); + Assert.Equal(1, manifest.ClaimCount); + + // second call hits cache + var cached = await engine.ExportAsync(context, CancellationToken.None); + Assert.True(cached.FromCache); + Assert.Equal(manifest.ExportId, cached.ExportId); + } + + [Fact] + public async Task ExportAsync_ForceRefreshInvalidatesCacheEntry() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var cacheIndex = new RecordingCacheIndex(); + var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger.Instance, cacheIndex); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var initialContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); + _ = await engine.ExportAsync(initialContext, CancellationToken.None); + + var refreshContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow.AddMinutes(1), ForceRefresh: true); + var refreshed = await engine.ExportAsync(refreshContext, CancellationToken.None); + + Assert.False(refreshed.FromCache); + var signature = VexQuerySignature.FromQuery(refreshContext.Query); + Assert.True(cacheIndex.RemoveCalls.TryGetValue((signature.Value, refreshContext.Format), out var removed)); + Assert.True(removed); + } + + [Fact] + public async Task ExportAsync_WritesArtifactsToAllStores() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var recorder1 = new RecordingArtifactStore(); + var recorder2 = new RecordingArtifactStore(); + var engine = new VexExportEngine( + store, + evaluator, + dataSource, + new[] { exporter }, + NullLogger.Instance, + cacheIndex: null, + artifactStores: new[] { recorder1, recorder2 }); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); + + await engine.ExportAsync(context, CancellationToken.None); + + Assert.Equal(1, recorder1.SaveCount); + Assert.Equal(1, recorder2.SaveCount); + } + + [Fact] + public async Task ExportAsync_AttachesAttestationMetadata() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var attestation = new RecordingAttestationClient(); + var engine = new VexExportEngine( + store, + evaluator, + dataSource, + new[] { exporter }, + NullLogger.Instance, + cacheIndex: null, + artifactStores: null, + attestationClient: attestation); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var requestedAt = DateTimeOffset.UtcNow; + var context = new VexExportRequestContext(query, VexExportFormat.Json, requestedAt); + + var manifest = await engine.ExportAsync(context, CancellationToken.None); + + Assert.NotNull(attestation.LastRequest); + Assert.Equal(manifest.ExportId, attestation.LastRequest!.ExportId); + Assert.NotNull(manifest.Attestation); + Assert.Equal(attestation.Response.Attestation.EnvelopeDigest, manifest.Attestation!.EnvelopeDigest); + Assert.Equal(attestation.Response.Attestation.PredicateType, manifest.Attestation.PredicateType); + + Assert.NotNull(store.LastSavedManifest); + Assert.Equal(manifest.Attestation, store.LastSavedManifest!.Attestation); + } + + private sealed class InMemoryExportStore : IVexExportStore + { + private readonly Dictionary _store = new(StringComparer.Ordinal); + + public VexExportManifest? LastSavedManifest { get; private set; } + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var key = CreateKey(signature.Value, format); + _store.TryGetValue(key, out var manifest); + return ValueTask.FromResult(manifest); + } + + public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var key = CreateKey(manifest.QuerySignature.Value, manifest.Format); + _store[key] = manifest; + LastSavedManifest = manifest; + return ValueTask.CompletedTask; + } + + private static string CreateKey(string signature, VexExportFormat format) + => FormattableString.Invariant($"{signature}|{format}"); + } + + private sealed class RecordingAttestationClient : IVexAttestationClient + { + public VexAttestationRequest? LastRequest { get; private set; } + + public VexAttestationResponse Response { get; } = new VexAttestationResponse( + new VexAttestationMetadata( + predicateType: "https://stella-ops.org/attestations/vex-export", + rekor: new VexRekorReference("0.2", "rekor://entry", "123"), + envelopeDigest: "sha256:envelope", + signedAt: DateTimeOffset.UnixEpoch), + ImmutableDictionary.Empty); + + public ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) + { + LastRequest = request; + return ValueTask.FromResult(Response); + } + + public ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary.Empty)); + } + + private sealed class RecordingCacheIndex : IVexCacheIndex + { + public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new(); + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(null); + + public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.CompletedTask; + + public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + RemoveCalls[(signature.Value, format)] = true; + return ValueTask.CompletedTask; + } + } + + private sealed class RecordingArtifactStore : IVexArtifactStore + { + public int SaveCount { get; private set; } + + public ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) + { + SaveCount++; + return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory", artifact.Content.Length, artifact.Metadata)); + } + + public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class StaticPolicyEvaluator : IVexPolicyEvaluator + { + public StaticPolicyEvaluator(string version) + { + Version = version; + } + + public string Version { get; } + + public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; + + public double GetProviderWeight(VexProvider provider) => 1.0; + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + rejectionReason = null; + return true; + } + } + + private sealed class InMemoryExportDataSource : IVexExportDataSource + { + public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) + { + var claim = new VexClaim( + "CVE-2025-0001", + "vendor", + new VexProduct("pkg:demo/app", "Demo"), + VexClaimStatus.Affected, + new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:demo", new Uri("https://example.org/demo")), + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow); + + var consensus = new VexConsensus( + "CVE-2025-0001", + claim.Product, + VexConsensusStatus.Affected, + DateTimeOffset.UtcNow, + new[] { new VexConsensusSource("vendor", VexClaimStatus.Affected, "sha256:demo", 1.0) }, + conflicts: null, + policyVersion: "baseline/v1", + summary: "affected"); + + return ValueTask.FromResult(new VexExportDataSet( + ImmutableArray.Create(consensus), + ImmutableArray.Create(claim), + ImmutableArray.Create("vendor"))); + } + } + + private sealed class DummyExporter : IVexExporter + { + public DummyExporter(VexExportFormat format) + { + Format = format; + } + + public VexExportFormat Format { get; } + + public VexContentAddress Digest(VexExportRequest request) + => new("sha256", "deadbeef"); + + public ValueTask SerializeAsync(VexExportRequest request, Stream output, CancellationToken cancellationToken) + { + var bytes = System.Text.Encoding.UTF8.GetBytes("{}"); + output.Write(bytes); + return ValueTask.FromResult(new VexExportResult(new VexContentAddress("sha256", "deadbeef"), bytes.Length, ImmutableDictionary.Empty)); + } + } + +} diff --git a/src/StellaOps.Excititor.Export.Tests/VexExportCacheServiceTests.cs b/src/StellaOps.Excititor.Export.Tests/VexExportCacheServiceTests.cs index 29c84f9f..2dc6c108 100644 --- a/src/StellaOps.Excititor.Export.Tests/VexExportCacheServiceTests.cs +++ b/src/StellaOps.Excititor.Export.Tests/VexExportCacheServiceTests.cs @@ -1,82 +1,82 @@ -using Microsoft.Extensions.Logging.Abstractions; -using MongoDB.Driver; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Export; -using StellaOps.Excititor.Storage.Mongo; - -namespace StellaOps.Excititor.Export.Tests; - -public sealed class VexExportCacheServiceTests -{ - [Fact] - public async Task InvalidateAsync_RemovesEntry() - { - var cacheIndex = new RecordingIndex(); - var maintenance = new StubMaintenance(); - var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); - - var signature = new VexQuerySignature("format=json|provider=vendor"); - await service.InvalidateAsync(signature, VexExportFormat.Json, CancellationToken.None); - - Assert.Equal(signature.Value, cacheIndex.LastSignature?.Value); - Assert.Equal(VexExportFormat.Json, cacheIndex.LastFormat); - Assert.Equal(1, cacheIndex.RemoveCalls); - } - - [Fact] - public async Task PruneExpiredAsync_ReturnsCount() - { - var cacheIndex = new RecordingIndex(); - var maintenance = new StubMaintenance { ExpiredCount = 3 }; - var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); - - var removed = await service.PruneExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None); - - Assert.Equal(3, removed); - } - - [Fact] - public async Task PruneDanglingAsync_ReturnsCount() - { - var cacheIndex = new RecordingIndex(); - var maintenance = new StubMaintenance { DanglingCount = 2 }; - var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); - - var removed = await service.PruneDanglingAsync(CancellationToken.None); - - Assert.Equal(2, removed); - } - - private sealed class RecordingIndex : IVexCacheIndex - { - public VexQuerySignature? LastSignature { get; private set; } - public VexExportFormat LastFormat { get; private set; } - public int RemoveCalls { get; private set; } - - public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - - public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) - => ValueTask.CompletedTask; - - public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - LastSignature = signature; - LastFormat = format; - RemoveCalls++; - return ValueTask.CompletedTask; - } - } - - private sealed class StubMaintenance : IVexCacheMaintenance - { - public int ExpiredCount { get; set; } - public int DanglingCount { get; set; } - - public ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) - => ValueTask.FromResult(ExpiredCount); - - public ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) - => ValueTask.FromResult(DanglingCount); - } -} +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Export.Tests; + +public sealed class VexExportCacheServiceTests +{ + [Fact] + public async Task InvalidateAsync_RemovesEntry() + { + var cacheIndex = new RecordingIndex(); + var maintenance = new StubMaintenance(); + var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); + + var signature = new VexQuerySignature("format=json|provider=vendor"); + await service.InvalidateAsync(signature, VexExportFormat.Json, CancellationToken.None); + + Assert.Equal(signature.Value, cacheIndex.LastSignature?.Value); + Assert.Equal(VexExportFormat.Json, cacheIndex.LastFormat); + Assert.Equal(1, cacheIndex.RemoveCalls); + } + + [Fact] + public async Task PruneExpiredAsync_ReturnsCount() + { + var cacheIndex = new RecordingIndex(); + var maintenance = new StubMaintenance { ExpiredCount = 3 }; + var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); + + var removed = await service.PruneExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None); + + Assert.Equal(3, removed); + } + + [Fact] + public async Task PruneDanglingAsync_ReturnsCount() + { + var cacheIndex = new RecordingIndex(); + var maintenance = new StubMaintenance { DanglingCount = 2 }; + var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); + + var removed = await service.PruneDanglingAsync(CancellationToken.None); + + Assert.Equal(2, removed); + } + + private sealed class RecordingIndex : IVexCacheIndex + { + public VexQuerySignature? LastSignature { get; private set; } + public VexExportFormat LastFormat { get; private set; } + public int RemoveCalls { get; private set; } + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(null); + + public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.CompletedTask; + + public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + LastSignature = signature; + LastFormat = format; + RemoveCalls++; + return ValueTask.CompletedTask; + } + } + + private sealed class StubMaintenance : IVexCacheMaintenance + { + public int ExpiredCount { get; set; } + public int DanglingCount { get; set; } + + public ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(ExpiredCount); + + public ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(DanglingCount); + } +} diff --git a/src/StellaOps.Excititor.Export/ExportEngine.cs b/src/StellaOps.Excititor.Export/ExportEngine.cs index 0c4397dc..8c5f6b32 100644 --- a/src/StellaOps.Excititor.Export/ExportEngine.cs +++ b/src/StellaOps.Excititor.Export/ExportEngine.cs @@ -1,209 +1,244 @@ -using System.Collections.Immutable; -using System.IO; -using System.Linq; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Policy; -using StellaOps.Excititor.Storage.Mongo; - -namespace StellaOps.Excititor.Export; - -public interface IExportEngine -{ - ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken); -} - -public sealed record VexExportRequestContext( - VexQuery Query, - VexExportFormat Format, - DateTimeOffset RequestedAt, - bool ForceRefresh = false); - -public interface IVexExportDataSource -{ - ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken); -} - -public sealed record VexExportDataSet( - ImmutableArray Consensus, - ImmutableArray Claims, - ImmutableArray SourceProviders); - -public sealed class VexExportEngine : IExportEngine -{ - private readonly IVexExportStore _exportStore; - private readonly IVexPolicyEvaluator _policyEvaluator; - private readonly IVexExportDataSource _dataSource; - private readonly IReadOnlyDictionary _exporters; - private readonly ILogger _logger; - private readonly IVexCacheIndex? _cacheIndex; - private readonly IReadOnlyList _artifactStores; - private readonly IVexAttestationClient? _attestationClient; - - public VexExportEngine( - IVexExportStore exportStore, - IVexPolicyEvaluator policyEvaluator, - IVexExportDataSource dataSource, - IEnumerable exporters, - ILogger logger, - IVexCacheIndex? cacheIndex = null, - IEnumerable? artifactStores = null, - IVexAttestationClient? attestationClient = null) - { - _exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore)); - _policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator)); - _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _cacheIndex = cacheIndex; - _artifactStores = artifactStores?.ToArray() ?? Array.Empty(); - _attestationClient = attestationClient; - - if (exporters is null) - { - throw new ArgumentNullException(nameof(exporters)); - } - - _exporters = exporters.ToDictionary(x => x.Format); - } - - public async ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - var signature = VexQuerySignature.FromQuery(context.Query); - - if (!context.ForceRefresh) - { - var cached = await _exportStore.FindAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); - if (cached is not null) - { - _logger.LogInformation("Reusing cached export for {Signature} ({Format})", signature.Value, context.Format); - return new VexExportManifest( - cached.ExportId, - cached.QuerySignature, - cached.Format, - cached.CreatedAt, - cached.Artifact, - cached.ClaimCount, - cached.SourceProviders, - fromCache: true, - cached.ConsensusRevision, - cached.Attestation, - cached.SizeBytes); - } - } - else if (_cacheIndex is not null) - { - await _cacheIndex.RemoveAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); - _logger.LogInformation("Force refresh requested; invalidated cache entry for {Signature} ({Format})", signature.Value, context.Format); - } - - var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false); - var exporter = ResolveExporter(context.Format); - - var exportRequest = new VexExportRequest( - context.Query, - dataset.Consensus, - dataset.Claims, - context.RequestedAt); - - var digest = exporter.Digest(exportRequest); - var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}"); - - await using var buffer = new MemoryStream(); - var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false); - - if (_artifactStores.Count > 0) - { - var writtenBytes = buffer.ToArray(); - try - { - var artifact = new VexExportArtifact( - result.Digest, - context.Format, - writtenBytes, - result.Metadata); - - foreach (var store in _artifactStores) - { - await store.SaveAsync(artifact, cancellationToken).ConfigureAwait(false); - } - - _logger.LogInformation("Stored export artifact {Digest} via {StoreCount} store(s)", result.Digest.ToUri(), _artifactStores.Count); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to store export artifact {Digest}", result.Digest.ToUri()); - throw; - } - } - - VexAttestationMetadata? attestationMetadata = null; - if (_attestationClient is not null) - { - var attestationRequest = new VexAttestationRequest( - exportId, - signature, - digest, - context.Format, - context.RequestedAt, - dataset.SourceProviders, - result.Metadata); - - var response = await _attestationClient.SignAsync(attestationRequest, cancellationToken).ConfigureAwait(false); - attestationMetadata = response.Attestation; - - if (!response.Diagnostics.IsEmpty) - { - foreach (var diagnostic in response.Diagnostics) - { - _logger.LogDebug( - "Attestation diagnostic {Key}={Value} for export {ExportId}", - diagnostic.Key, - diagnostic.Value, - exportId); - } - } - - _logger.LogInformation("Attestation generated for export {ExportId}", exportId); - } - - var manifest = new VexExportManifest( - exportId, - signature, - context.Format, - context.RequestedAt, - digest, - dataset.Claims.Length, - dataset.SourceProviders, - fromCache: false, - consensusRevision: _policyEvaluator.Version, - attestation: attestationMetadata, - sizeBytes: result.BytesWritten); - - await _exportStore.SaveAsync(manifest, cancellationToken).ConfigureAwait(false); - - _logger.LogInformation( - "Export generated for {Signature} ({Format}) size={SizeBytes} bytes", - signature.Value, - context.Format, - result.BytesWritten); - - return manifest; - } - - private IVexExporter ResolveExporter(VexExportFormat format) - => _exporters.TryGetValue(format, out var exporter) - ? exporter - : throw new InvalidOperationException($"No exporter registered for format '{format}'."); -} - -public static class VexExportServiceCollectionExtensions -{ - public static IServiceCollection AddVexExportEngine(this IServiceCollection services) - { - services.AddSingleton(); - services.AddVexExportCacheServices(); - return services; - } -} +using System; +using System.Collections.Immutable; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Export; + +public interface IExportEngine +{ + ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken); +} + +public sealed record VexExportRequestContext( + VexQuery Query, + VexExportFormat Format, + DateTimeOffset RequestedAt, + bool ForceRefresh = false); + +public interface IVexExportDataSource +{ + ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken); +} + +public sealed record VexExportDataSet( + ImmutableArray Consensus, + ImmutableArray Claims, + ImmutableArray SourceProviders); + +public sealed class VexExportEngine : IExportEngine +{ + private readonly IVexExportStore _exportStore; + private readonly IVexPolicyEvaluator _policyEvaluator; + private readonly IVexExportDataSource _dataSource; + private readonly IReadOnlyDictionary _exporters; + private readonly ILogger _logger; + private readonly IVexCacheIndex? _cacheIndex; + private readonly IReadOnlyList _artifactStores; + private readonly IVexAttestationClient? _attestationClient; + + public VexExportEngine( + IVexExportStore exportStore, + IVexPolicyEvaluator policyEvaluator, + IVexExportDataSource dataSource, + IEnumerable exporters, + ILogger logger, + IVexCacheIndex? cacheIndex = null, + IEnumerable? artifactStores = null, + IVexAttestationClient? attestationClient = null) + { + _exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore)); + _policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator)); + _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _cacheIndex = cacheIndex; + _artifactStores = artifactStores?.ToArray() ?? Array.Empty(); + _attestationClient = attestationClient; + + if (exporters is null) + { + throw new ArgumentNullException(nameof(exporters)); + } + + _exporters = exporters.ToDictionary(x => x.Format); + } + + public async ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + var signature = VexQuerySignature.FromQuery(context.Query); + + if (!context.ForceRefresh) + { + var cached = await _exportStore.FindAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); + if (cached is not null) + { + _logger.LogInformation("Reusing cached export for {Signature} ({Format})", signature.Value, context.Format); + return new VexExportManifest( + cached.ExportId, + cached.QuerySignature, + cached.Format, + cached.CreatedAt, + cached.Artifact, + cached.ClaimCount, + cached.SourceProviders, + fromCache: true, + cached.ConsensusRevision, + cached.PolicyRevisionId, + cached.PolicyDigest, + cached.ConsensusDigest, + cached.ScoreDigest, + cached.Attestation, + cached.SizeBytes); + } + } + else if (_cacheIndex is not null) + { + await _cacheIndex.RemoveAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Force refresh requested; invalidated cache entry for {Signature} ({Format})", signature.Value, context.Format); + } + + var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false); + var exporter = ResolveExporter(context.Format); + var policySnapshot = _policyEvaluator.Snapshot; + + var exportRequest = new VexExportRequest( + context.Query, + dataset.Consensus, + dataset.Claims, + context.RequestedAt); + + var digest = exporter.Digest(exportRequest); + var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}"); + + await using var buffer = new MemoryStream(); + var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false); + + if (_artifactStores.Count > 0) + { + var writtenBytes = buffer.ToArray(); + try + { + var artifact = new VexExportArtifact( + result.Digest, + context.Format, + writtenBytes, + result.Metadata); + + foreach (var store in _artifactStores) + { + await store.SaveAsync(artifact, cancellationToken).ConfigureAwait(false); + } + + _logger.LogInformation("Stored export artifact {Digest} via {StoreCount} store(s)", result.Digest.ToUri(), _artifactStores.Count); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to store export artifact {Digest}", result.Digest.ToUri()); + throw; + } + } + + VexAttestationMetadata? attestationMetadata = null; + if (_attestationClient is not null) + { + var attestationRequest = new VexAttestationRequest( + exportId, + signature, + digest, + context.Format, + context.RequestedAt, + dataset.SourceProviders, + result.Metadata); + + var response = await _attestationClient.SignAsync(attestationRequest, cancellationToken).ConfigureAwait(false); + attestationMetadata = response.Attestation; + + if (!response.Diagnostics.IsEmpty) + { + foreach (var diagnostic in response.Diagnostics) + { + _logger.LogDebug( + "Attestation diagnostic {Key}={Value} for export {ExportId}", + diagnostic.Key, + diagnostic.Value, + exportId); + } + } + + _logger.LogInformation("Attestation generated for export {ExportId}", exportId); + } + + var consensusDigestAddress = TryGetContentAddress(result.Metadata, "consensusDigest"); + var scoreDigestAddress = TryGetContentAddress(result.Metadata, "scoreDigest"); + + var manifest = new VexExportManifest( + exportId, + signature, + context.Format, + context.RequestedAt, + digest, + dataset.Claims.Length, + dataset.SourceProviders, + fromCache: false, + consensusRevision: policySnapshot.Version, + policyRevisionId: policySnapshot.RevisionId, + policyDigest: policySnapshot.Digest, + consensusDigest: consensusDigestAddress, + scoreDigest: scoreDigestAddress, + attestation: attestationMetadata, + sizeBytes: result.BytesWritten); + + await _exportStore.SaveAsync(manifest, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Export generated for {Signature} ({Format}) size={SizeBytes} bytes", + signature.Value, + context.Format, + result.BytesWritten); + + return manifest; + } + + private static VexContentAddress? TryGetContentAddress(IReadOnlyDictionary metadata, string key) + { + if (metadata is null) + { + return null; + } + + if (!metadata.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var parts = value.Split(':', 2, StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + return null; + } + + return new VexContentAddress(parts[0], parts[1]); + } + + private IVexExporter ResolveExporter(VexExportFormat format) + => _exporters.TryGetValue(format, out var exporter) + ? exporter + : throw new InvalidOperationException($"No exporter registered for format '{format}'."); +} + +public static class VexExportServiceCollectionExtensions +{ + public static IServiceCollection AddVexExportEngine(this IServiceCollection services) + { + services.AddSingleton(); + services.AddVexExportCacheServices(); + return services; + } +} diff --git a/src/StellaOps.Excititor.Formats.CSAF/CsafNormalizer.cs b/src/StellaOps.Excititor.Formats.CSAF/CsafNormalizer.cs index 11cb0978..e68b5fd8 100644 --- a/src/StellaOps.Excititor.Formats.CSAF/CsafNormalizer.cs +++ b/src/StellaOps.Excititor.Formats.CSAF/CsafNormalizer.cs @@ -1,875 +1,879 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Excititor.Core; - -namespace StellaOps.Excititor.Formats.CSAF; - -public sealed class CsafNormalizer : IVexNormalizer -{ - private static readonly ImmutableDictionary StatusPrecedence = new Dictionary - { - [VexClaimStatus.UnderInvestigation] = 0, - [VexClaimStatus.Affected] = 1, - [VexClaimStatus.NotAffected] = 2, - [VexClaimStatus.Fixed] = 3, - }.ToImmutableDictionary(); - - private static readonly ImmutableDictionary StatusMap = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["known_affected"] = VexClaimStatus.Affected, - ["first_affected"] = VexClaimStatus.Affected, - ["last_affected"] = VexClaimStatus.Affected, - ["affected"] = VexClaimStatus.Affected, - ["fixed_after_release"] = VexClaimStatus.Fixed, - ["fixed"] = VexClaimStatus.Fixed, - ["first_fixed"] = VexClaimStatus.Fixed, - ["last_fixed"] = VexClaimStatus.Fixed, - ["recommended"] = VexClaimStatus.Fixed, - ["known_not_affected"] = VexClaimStatus.NotAffected, - ["first_not_affected"] = VexClaimStatus.NotAffected, - ["last_not_affected"] = VexClaimStatus.NotAffected, - ["not_affected"] = VexClaimStatus.NotAffected, - ["under_investigation"] = VexClaimStatus.UnderInvestigation, - ["investigating"] = VexClaimStatus.UnderInvestigation, - ["in_investigation"] = VexClaimStatus.UnderInvestigation, - ["in_triage"] = VexClaimStatus.UnderInvestigation, - ["unknown"] = VexClaimStatus.UnderInvestigation, - }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - - private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["component_not_present"] = VexJustification.ComponentNotPresent, - ["component_not_configured"] = VexJustification.ComponentNotConfigured, - ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, - ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, - ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, - ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, - ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, - ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, - ["protected_at_runtime"] = VexJustification.ProtectedAtRuntime, - ["protected_at_perimeter"] = VexJustification.ProtectedAtPerimeter, - ["code_not_present"] = VexJustification.CodeNotPresent, - ["code_not_reachable"] = VexJustification.CodeNotReachable, - ["requires_configuration"] = VexJustification.RequiresConfiguration, - ["requires_dependency"] = VexJustification.RequiresDependency, - ["requires_environment"] = VexJustification.RequiresEnvironment, - }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - - private readonly ILogger _logger; - - public CsafNormalizer(ILogger logger) - { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string Format => VexDocumentFormat.Csaf.ToString().ToLowerInvariant(); - - public bool CanHandle(VexRawDocument document) - => document is not null && document.Format == VexDocumentFormat.Csaf; - - public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(provider); - - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var result = CsafParser.Parse(document); - var claims = ImmutableArray.CreateBuilder(result.Claims.Length); - foreach (var entry in result.Claims) - { - var product = new VexProduct( - entry.Product.ProductId, - entry.Product.Name, - entry.Product.Version, - entry.Product.Purl, - entry.Product.Cpe); - - var claimDocument = new VexClaimDocument( - VexDocumentFormat.Csaf, - document.Digest, - document.SourceUri, - result.Revision, - signature: null); - - var metadata = result.Metadata; - if (!string.IsNullOrWhiteSpace(entry.RawStatus)) - { - metadata = metadata.SetItem("csaf.product_status.raw", entry.RawStatus); - } - - if (!string.IsNullOrWhiteSpace(entry.RawJustification)) - { - metadata = metadata.SetItem("csaf.justification.label", entry.RawJustification); - } - - var claim = new VexClaim( - entry.VulnerabilityId, - provider.Id, - product, - entry.Status, - claimDocument, - result.FirstRelease, - result.LastRelease, - entry.Justification, - detail: entry.Detail, - confidence: null, - additionalMetadata: metadata); - - claims.Add(claim); - } - - var orderedClaims = claims - .ToImmutable() - .OrderBy(static claim => claim.VulnerabilityId, StringComparer.Ordinal) - .ThenBy(static claim => claim.Product.Key, StringComparer.Ordinal) - .ToImmutableArray(); - - _logger.LogInformation( - "Normalized CSAF document {Source} into {ClaimCount} claim(s).", - document.SourceUri, - orderedClaims.Length); - - var diagnosticsBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - if (!result.UnsupportedStatuses.IsDefaultOrEmpty && result.UnsupportedStatuses.Length > 0) - { - diagnosticsBuilder["csaf.unsupported_statuses"] = string.Join(",", result.UnsupportedStatuses); - } - - if (!result.UnsupportedJustifications.IsDefaultOrEmpty && result.UnsupportedJustifications.Length > 0) - { - diagnosticsBuilder["csaf.unsupported_justifications"] = string.Join(",", result.UnsupportedJustifications); - } - - if (!result.ConflictingJustifications.IsDefaultOrEmpty && result.ConflictingJustifications.Length > 0) - { - diagnosticsBuilder["csaf.justification_conflicts"] = string.Join(",", result.ConflictingJustifications); - } - - var diagnostics = diagnosticsBuilder.Count == 0 - ? ImmutableDictionary.Empty - : diagnosticsBuilder.ToImmutable(); - - return ValueTask.FromResult(new VexClaimBatch(document, orderedClaims, diagnostics)); - } - catch (JsonException ex) - { - _logger.LogError(ex, "Failed to parse CSAF document {SourceUri}", document.SourceUri); - throw; - } - } - - private static class CsafParser - { - public static CsafParseResult Parse(VexRawDocument document) - { - using var json = JsonDocument.Parse(document.Content.ToArray()); - var root = json.RootElement; - - var tracking = root.TryGetProperty("document", out var documentElement) && - documentElement.ValueKind == JsonValueKind.Object && - documentElement.TryGetProperty("tracking", out var trackingElement) - ? trackingElement - : default; - - var firstRelease = ParseDate(tracking, "initial_release_date") ?? document.RetrievedAt; - var lastRelease = ParseDate(tracking, "current_release_date") ?? firstRelease; - - if (lastRelease < firstRelease) - { - lastRelease = firstRelease; - } - - var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - AddIfPresent(metadataBuilder, tracking, "id", "csaf.tracking.id"); - AddIfPresent(metadataBuilder, tracking, "version", "csaf.tracking.version"); - AddIfPresent(metadataBuilder, tracking, "status", "csaf.tracking.status"); - AddPublisherMetadata(metadataBuilder, documentElement); - - var revision = TryGetString(tracking, "revision"); - - var productCatalog = CollectProducts(root); - var productGroups = CollectProductGroups(root); - - var unsupportedStatuses = new HashSet(StringComparer.OrdinalIgnoreCase); - var unsupportedJustifications = new HashSet(StringComparer.OrdinalIgnoreCase); - var conflictingJustifications = new HashSet(StringComparer.OrdinalIgnoreCase); - - var claimsBuilder = ImmutableArray.CreateBuilder(); - - if (root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) && - vulnerabilitiesElement.ValueKind == JsonValueKind.Array) - { - foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) - { - var vulnerabilityId = ResolveVulnerabilityId(vulnerability); - if (string.IsNullOrWhiteSpace(vulnerabilityId)) - { - continue; - } - - var detail = ResolveDetail(vulnerability); - var justifications = CollectJustifications( - vulnerability, - productCatalog, - productGroups, - unsupportedJustifications, - conflictingJustifications); - - var productClaims = BuildClaimsForVulnerability( - vulnerabilityId, - vulnerability, - productCatalog, - justifications, - detail, - unsupportedStatuses); - - claimsBuilder.AddRange(productClaims); - } - } - - return new CsafParseResult( - firstRelease, - lastRelease, - revision, - metadataBuilder.ToImmutable(), - claimsBuilder.ToImmutable(), - unsupportedStatuses.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray(), - unsupportedJustifications.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray(), - conflictingJustifications.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); - } - - private static IReadOnlyList BuildClaimsForVulnerability( - string vulnerabilityId, - JsonElement vulnerability, - IReadOnlyDictionary productCatalog, - ImmutableDictionary justifications, - string? detail, - ISet unsupportedStatuses) - { - if (!vulnerability.TryGetProperty("product_status", out var statusElement) || - statusElement.ValueKind != JsonValueKind.Object) - { - return Array.Empty(); - } - - var claims = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var statusProperty in statusElement.EnumerateObject()) - { - var status = MapStatus(statusProperty.Name, unsupportedStatuses); - if (status is null) - { - continue; - } - - if (statusProperty.Value.ValueKind != JsonValueKind.Array) - { - continue; - } - - foreach (var productIdElement in statusProperty.Value.EnumerateArray()) - { - var productId = productIdElement.GetString(); - if (string.IsNullOrWhiteSpace(productId)) - { - continue; - } - - var trimmedProductId = productId.Trim(); - var product = ResolveProduct(productCatalog, trimmedProductId); - justifications.TryGetValue(trimmedProductId, out var justificationInfo); - - UpdateClaim(claims, product, status.Value, statusProperty.Name, detail, justificationInfo); - } - } - - if (claims.Count == 0) - { - return Array.Empty(); - } - - return claims.Values - .Select(builder => new CsafClaimEntry( - vulnerabilityId, - builder.Product, - builder.Status, - builder.RawStatus, - builder.Detail, - builder.Justification, - builder.RawJustification)) - .ToArray(); - } - - private static void UpdateClaim( - IDictionary claims, - CsafProductInfo product, - VexClaimStatus status, - string rawStatus, - string? detail, - CsafJustificationInfo? justification) - { - if (!claims.TryGetValue(product.ProductId, out var existing) || - StatusPrecedence[status] > StatusPrecedence[existing.Status]) - { - claims[product.ProductId] = new CsafClaimEntryBuilder( - product, - status, - NormalizeRaw(rawStatus), - detail, - justification?.Normalized, - justification?.RawValue); - return; - } - - if (StatusPrecedence[status] < StatusPrecedence[existing.Status]) - { - return; - } - - var updated = existing; - - if (string.IsNullOrWhiteSpace(existing.RawStatus)) - { - updated = updated with { RawStatus = NormalizeRaw(rawStatus) }; - } - - if (existing.Detail is null && detail is not null) - { - updated = updated with { Detail = detail }; - } - - if (justification is not null) - { - if (existing.Justification is null && justification.Normalized is not null) - { - updated = updated with - { - Justification = justification.Normalized, - RawJustification = justification.RawValue - }; - } - else if (existing.Justification is null && - justification.Normalized is null && - string.IsNullOrWhiteSpace(existing.RawJustification) && - !string.IsNullOrWhiteSpace(justification.RawValue)) - { - updated = updated with { RawJustification = justification.RawValue }; - } - } - - claims[product.ProductId] = updated; - } - - private static CsafProductInfo ResolveProduct( - IReadOnlyDictionary catalog, - string productId) - { - if (catalog.TryGetValue(productId, out var product)) - { - return product; - } - - return new CsafProductInfo(productId, productId, null, null, null); - } - - private static string ResolveVulnerabilityId(JsonElement vulnerability) - { - var id = TryGetString(vulnerability, "cve") - ?? TryGetString(vulnerability, "id") - ?? TryGetString(vulnerability, "vuln_id"); - - return string.IsNullOrWhiteSpace(id) ? string.Empty : id.Trim(); - } - - private static string? ResolveDetail(JsonElement vulnerability) - { - var title = TryGetString(vulnerability, "title"); - if (!string.IsNullOrWhiteSpace(title)) - { - return title.Trim(); - } - - if (vulnerability.TryGetProperty("notes", out var notesElement) && - notesElement.ValueKind == JsonValueKind.Array) - { - foreach (var note in notesElement.EnumerateArray()) - { - if (note.ValueKind != JsonValueKind.Object) - { - continue; - } - - var category = TryGetString(note, "category"); - if (!string.IsNullOrWhiteSpace(category) && - !string.Equals(category, "description", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var text = TryGetString(note, "text"); - if (!string.IsNullOrWhiteSpace(text)) - { - return text.Trim(); - } - } - } - - return null; - } - - private static Dictionary CollectProducts(JsonElement root) - { - var products = new Dictionary(StringComparer.OrdinalIgnoreCase); - - if (!root.TryGetProperty("product_tree", out var productTree) || - productTree.ValueKind != JsonValueKind.Object) - { - return products; - } - - if (productTree.TryGetProperty("full_product_names", out var fullNames) && - fullNames.ValueKind == JsonValueKind.Array) - { - foreach (var productEntry in fullNames.EnumerateArray()) - { - var product = ParseProduct(productEntry, parentBranchName: null); - if (product is not null) - { - AddOrUpdate(product); - } - } - } - - if (productTree.TryGetProperty("branches", out var branches) && - branches.ValueKind == JsonValueKind.Array) - { - foreach (var branch in branches.EnumerateArray()) - { - VisitBranch(branch, parentBranchName: null); - } - } - - return products; - - void VisitBranch(JsonElement branch, string? parentBranchName) - { - if (branch.ValueKind != JsonValueKind.Object) - { - return; - } - - var branchName = TryGetString(branch, "name") ?? parentBranchName; - - if (branch.TryGetProperty("product", out var productElement)) - { - var product = ParseProduct(productElement, branchName); - if (product is not null) - { - AddOrUpdate(product); - } - } - - if (branch.TryGetProperty("branches", out var childBranches) && - childBranches.ValueKind == JsonValueKind.Array) - { - foreach (var childBranch in childBranches.EnumerateArray()) - { - VisitBranch(childBranch, branchName); - } - } - } - - void AddOrUpdate(CsafProductInfo product) - { - if (products.TryGetValue(product.ProductId, out var existing)) - { - products[product.ProductId] = MergeProducts(existing, product); - } - else - { - products[product.ProductId] = product; - } - } - - static CsafProductInfo MergeProducts(CsafProductInfo existing, CsafProductInfo incoming) - { - static string ChooseName(string incoming, string fallback) - => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; - - static string? ChooseOptional(string? incoming, string? fallback) - => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; - - return new CsafProductInfo( - existing.ProductId, - ChooseName(incoming.Name, existing.Name), - ChooseOptional(incoming.Version, existing.Version), - ChooseOptional(incoming.Purl, existing.Purl), - ChooseOptional(incoming.Cpe, existing.Cpe)); - } - } - - private static ImmutableDictionary> CollectProductGroups(JsonElement root) - { - if (!root.TryGetProperty("product_tree", out var productTree) || - productTree.ValueKind != JsonValueKind.Object || - !productTree.TryGetProperty("product_groups", out var groupsElement) || - groupsElement.ValueKind != JsonValueKind.Array) - { - return ImmutableDictionary>.Empty; - } - - var groups = new Dictionary>(StringComparer.OrdinalIgnoreCase); - - foreach (var group in groupsElement.EnumerateArray()) - { - if (group.ValueKind != JsonValueKind.Object) - { - continue; - } - - var groupId = TryGetString(group, "group_id"); - if (string.IsNullOrWhiteSpace(groupId)) - { - continue; - } - - if (!group.TryGetProperty("product_ids", out var productIdsElement) || - productIdsElement.ValueKind != JsonValueKind.Array) - { - continue; - } - - var members = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var productIdElement in productIdsElement.EnumerateArray()) - { - var productId = productIdElement.GetString(); - if (string.IsNullOrWhiteSpace(productId)) - { - continue; - } - - members.Add(productId.Trim()); - } - - if (members.Count == 0) - { - continue; - } - - groups[groupId.Trim()] = members - .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - } - - return groups.Count == 0 - ? ImmutableDictionary>.Empty - : groups.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - } - - private static ImmutableDictionary CollectJustifications( - JsonElement vulnerability, - IReadOnlyDictionary productCatalog, - ImmutableDictionary> productGroups, - ISet unsupportedJustifications, - ISet conflictingJustifications) - { - if (!vulnerability.TryGetProperty("flags", out var flagsElement) || - flagsElement.ValueKind != JsonValueKind.Array) - { - return ImmutableDictionary.Empty; - } - - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var flag in flagsElement.EnumerateArray()) - { - if (flag.ValueKind != JsonValueKind.Object) - { - continue; - } - - var label = TryGetString(flag, "label"); - if (string.IsNullOrWhiteSpace(label)) - { - continue; - } - - var rawLabel = NormalizeRaw(label); - var normalized = MapJustification(rawLabel, unsupportedJustifications); - - var targetIds = ExpandFlagProducts(flag, productGroups); - foreach (var productId in targetIds) - { - if (!productCatalog.ContainsKey(productId)) - { - continue; - } - - var info = new CsafJustificationInfo(rawLabel, normalized); - if (map.TryGetValue(productId, out var existing)) - { - if (existing.Normalized is null && normalized is not null) - { - map[productId] = info; - } - else if (existing.Normalized is not null && normalized is not null && existing.Normalized != normalized) - { - conflictingJustifications.Add(productId); - } - else if (existing.Normalized is null && - normalized is null && - string.IsNullOrWhiteSpace(existing.RawValue) && - !string.IsNullOrWhiteSpace(rawLabel)) - { - map[productId] = info; - } - } - else - { - map[productId] = info; - } - } - } - - return map.Count == 0 - ? ImmutableDictionary.Empty - : map.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - } - - private static IEnumerable ExpandFlagProducts( - JsonElement flag, - ImmutableDictionary> productGroups) - { - var productIds = new HashSet(StringComparer.OrdinalIgnoreCase); - - if (flag.TryGetProperty("product_ids", out var productIdsElement) && - productIdsElement.ValueKind == JsonValueKind.Array) - { - foreach (var idElement in productIdsElement.EnumerateArray()) - { - var id = idElement.GetString(); - if (string.IsNullOrWhiteSpace(id)) - { - continue; - } - - productIds.Add(id.Trim()); - } - } - - if (flag.TryGetProperty("group_ids", out var groupIdsElement) && - groupIdsElement.ValueKind == JsonValueKind.Array) - { - foreach (var groupIdElement in groupIdsElement.EnumerateArray()) - { - var groupId = groupIdElement.GetString(); - if (string.IsNullOrWhiteSpace(groupId)) - { - continue; - } - - if (productGroups.TryGetValue(groupId.Trim(), out var members)) - { - foreach (var member in members) - { - productIds.Add(member); - } - } - } - } - - return productIds; - } - - private static VexJustification? MapJustification(string justification, ISet unsupportedJustifications) - { - if (string.IsNullOrWhiteSpace(justification)) - { - return null; - } - - if (JustificationMap.TryGetValue(justification, out var mapped)) - { - return mapped; - } - - unsupportedJustifications.Add(justification); - return null; - } - - private static string NormalizeRaw(string value) - => string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim(); - - private static CsafProductInfo? ParseProduct(JsonElement element, string? parentBranchName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - JsonElement productElement = element; - if (!element.TryGetProperty("product_id", out var idElement) && - element.TryGetProperty("product", out var nestedProduct) && - nestedProduct.ValueKind == JsonValueKind.Object && - nestedProduct.TryGetProperty("product_id", out idElement)) - { - productElement = nestedProduct; - } - - var productId = idElement.GetString(); - if (string.IsNullOrWhiteSpace(productId)) - { - return null; - } - - var name = TryGetString(productElement, "name") - ?? TryGetString(element, "name") - ?? parentBranchName - ?? productId; - - var version = TryGetString(productElement, "product_version") - ?? TryGetString(productElement, "version") - ?? TryGetString(element, "product_version"); - - string? cpe = null; - string? purl = null; - if (productElement.TryGetProperty("product_identification_helper", out var helper) && - helper.ValueKind == JsonValueKind.Object) - { - cpe = TryGetString(helper, "cpe"); - purl = TryGetString(helper, "purl"); - } - - return new CsafProductInfo(productId.Trim(), name.Trim(), version?.Trim(), purl?.Trim(), cpe?.Trim()); - } - - private static VexClaimStatus? MapStatus(string statusName, ISet unsupportedStatuses) - { - if (string.IsNullOrWhiteSpace(statusName)) - { - return null; - } - - var normalized = statusName.Trim(); - if (StatusMap.TryGetValue(normalized, out var mapped)) - { - return mapped; - } - - unsupportedStatuses.Add(normalized); - return null; - } - - private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var dateElement)) - { - return null; - } - - var value = dateElement.GetString(); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (DateTimeOffset.TryParse(value, out var parsed)) - { - return parsed; - } - - return null; - } - - private static string? TryGetString(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var property)) - { - return null; - } - - return property.ValueKind == JsonValueKind.String ? property.GetString() : null; - } - - private static void AddIfPresent( - ImmutableDictionary.Builder builder, - JsonElement element, - string propertyName, - string metadataKey) - { - var value = TryGetString(element, propertyName); - if (!string.IsNullOrWhiteSpace(value)) - { - builder[metadataKey] = value.Trim(); - } - } - - private static void AddPublisherMetadata( - ImmutableDictionary.Builder builder, - JsonElement documentElement) - { - if (documentElement.ValueKind != JsonValueKind.Object || - !documentElement.TryGetProperty("publisher", out var publisher) || - publisher.ValueKind != JsonValueKind.Object) - { - return; - } - - AddIfPresent(builder, publisher, "name", "csaf.publisher.name"); - AddIfPresent(builder, publisher, "category", "csaf.publisher.category"); - } - - private readonly record struct CsafClaimEntryBuilder( - CsafProductInfo Product, - VexClaimStatus Status, - string RawStatus, - string? Detail, - VexJustification? Justification, - string? RawJustification); - } - - private sealed record CsafParseResult( - DateTimeOffset FirstRelease, - DateTimeOffset LastRelease, - string? Revision, - ImmutableDictionary Metadata, - ImmutableArray Claims, - ImmutableArray UnsupportedStatuses, - ImmutableArray UnsupportedJustifications, - ImmutableArray ConflictingJustifications); - - private sealed record CsafClaimEntry( - string VulnerabilityId, - CsafProductInfo Product, - VexClaimStatus Status, - string RawStatus, - string? Detail, - VexJustification? Justification, - string? RawJustification); - - private sealed record CsafProductInfo( - string ProductId, - string Name, - string? Version, - string? Purl, - string? Cpe); -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Formats.CSAF; + +public sealed class CsafNormalizer : IVexNormalizer +{ + private static readonly ImmutableDictionary StatusPrecedence = new Dictionary + { + [VexClaimStatus.UnderInvestigation] = 0, + [VexClaimStatus.Affected] = 1, + [VexClaimStatus.NotAffected] = 2, + [VexClaimStatus.Fixed] = 3, + }.ToImmutableDictionary(); + + private static readonly ImmutableDictionary StatusMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["known_affected"] = VexClaimStatus.Affected, + ["first_affected"] = VexClaimStatus.Affected, + ["last_affected"] = VexClaimStatus.Affected, + ["affected"] = VexClaimStatus.Affected, + ["fixed_after_release"] = VexClaimStatus.Fixed, + ["fixed"] = VexClaimStatus.Fixed, + ["first_fixed"] = VexClaimStatus.Fixed, + ["last_fixed"] = VexClaimStatus.Fixed, + ["recommended"] = VexClaimStatus.Fixed, + ["known_not_affected"] = VexClaimStatus.NotAffected, + ["first_not_affected"] = VexClaimStatus.NotAffected, + ["last_not_affected"] = VexClaimStatus.NotAffected, + ["not_affected"] = VexClaimStatus.NotAffected, + ["under_investigation"] = VexClaimStatus.UnderInvestigation, + ["investigating"] = VexClaimStatus.UnderInvestigation, + ["in_investigation"] = VexClaimStatus.UnderInvestigation, + ["in_triage"] = VexClaimStatus.UnderInvestigation, + ["unknown"] = VexClaimStatus.UnderInvestigation, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["component_not_present"] = VexJustification.ComponentNotPresent, + ["component_not_configured"] = VexJustification.ComponentNotConfigured, + ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, + ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, + ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, + ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, + ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, + ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, + ["protected_at_runtime"] = VexJustification.ProtectedAtRuntime, + ["protected_at_perimeter"] = VexJustification.ProtectedAtPerimeter, + ["code_not_present"] = VexJustification.CodeNotPresent, + ["code_not_reachable"] = VexJustification.CodeNotReachable, + ["requires_configuration"] = VexJustification.RequiresConfiguration, + ["requires_dependency"] = VexJustification.RequiresDependency, + ["requires_environment"] = VexJustification.RequiresEnvironment, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private readonly ILogger _logger; + + public CsafNormalizer(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Format => VexDocumentFormat.Csaf.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) + => document is not null && document.Format == VexDocumentFormat.Csaf; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(provider); + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var result = CsafParser.Parse(document); + var claims = ImmutableArray.CreateBuilder(result.Claims.Length); + foreach (var entry in result.Claims) + { + var product = new VexProduct( + entry.Product.ProductId, + entry.Product.Name, + entry.Product.Version, + entry.Product.Purl, + entry.Product.Cpe); + + var claimDocument = new VexClaimDocument( + VexDocumentFormat.Csaf, + document.Digest, + document.SourceUri, + result.Revision, + signature: null); + + var metadata = result.Metadata; + if (!string.IsNullOrWhiteSpace(entry.RawStatus)) + { + metadata = metadata.SetItem("csaf.product_status.raw", entry.RawStatus); + } + + if (!string.IsNullOrWhiteSpace(entry.RawJustification)) + { + metadata = metadata.SetItem("csaf.justification.label", entry.RawJustification); + } + + var claim = new VexClaim( + entry.VulnerabilityId, + provider.Id, + product, + entry.Status, + claimDocument, + result.FirstRelease, + result.LastRelease, + entry.Justification, + detail: entry.Detail, + confidence: null, + additionalMetadata: metadata); + + claims.Add(claim); + } + + var orderedClaims = claims + .ToImmutable() + .OrderBy(static claim => claim.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(static claim => claim.Product.Key, StringComparer.Ordinal) + .ToImmutableArray(); + + _logger.LogInformation( + "Normalized CSAF document {Source} into {ClaimCount} claim(s).", + document.SourceUri, + orderedClaims.Length); + + var diagnosticsBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + if (!result.UnsupportedStatuses.IsDefaultOrEmpty && result.UnsupportedStatuses.Length > 0) + { + diagnosticsBuilder["csaf.unsupported_statuses"] = string.Join(",", result.UnsupportedStatuses); + } + + if (!result.UnsupportedJustifications.IsDefaultOrEmpty && result.UnsupportedJustifications.Length > 0) + { + diagnosticsBuilder["csaf.unsupported_justifications"] = string.Join(",", result.UnsupportedJustifications); + } + + if (!result.ConflictingJustifications.IsDefaultOrEmpty && result.ConflictingJustifications.Length > 0) + { + diagnosticsBuilder["csaf.justification_conflicts"] = string.Join(",", result.ConflictingJustifications); + } + + var diagnostics = diagnosticsBuilder.Count == 0 + ? ImmutableDictionary.Empty + : diagnosticsBuilder.ToImmutable(); + + return ValueTask.FromResult(new VexClaimBatch(document, orderedClaims, diagnostics)); + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to parse CSAF document {SourceUri}", document.SourceUri); + throw; + } + } + + private static class CsafParser + { + public static CsafParseResult Parse(VexRawDocument document) + { + using var json = JsonDocument.Parse(document.Content.ToArray()); + var root = json.RootElement; + + var tracking = root.TryGetProperty("document", out var documentElement) && + documentElement.ValueKind == JsonValueKind.Object && + documentElement.TryGetProperty("tracking", out var trackingElement) + ? trackingElement + : default; + + var firstRelease = ParseDate(tracking, "initial_release_date") ?? document.RetrievedAt; + var lastRelease = ParseDate(tracking, "current_release_date") ?? firstRelease; + + if (lastRelease < firstRelease) + { + lastRelease = firstRelease; + } + + var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + AddIfPresent(metadataBuilder, tracking, "id", "csaf.tracking.id"); + AddIfPresent(metadataBuilder, tracking, "version", "csaf.tracking.version"); + AddIfPresent(metadataBuilder, tracking, "status", "csaf.tracking.status"); + AddPublisherMetadata(metadataBuilder, documentElement); + + var revision = TryGetString(tracking, "revision"); + + var productCatalog = CollectProducts(root); + var productGroups = CollectProductGroups(root); + + var unsupportedStatuses = new HashSet(StringComparer.OrdinalIgnoreCase); + var unsupportedJustifications = new HashSet(StringComparer.OrdinalIgnoreCase); + var conflictingJustifications = new HashSet(StringComparer.OrdinalIgnoreCase); + + var claimsBuilder = ImmutableArray.CreateBuilder(); + + if (root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) && + vulnerabilitiesElement.ValueKind == JsonValueKind.Array) + { + foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) + { + var vulnerabilityId = ResolveVulnerabilityId(vulnerability); + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + continue; + } + + var detail = ResolveDetail(vulnerability); + var justifications = CollectJustifications( + vulnerability, + productCatalog, + productGroups, + unsupportedJustifications, + conflictingJustifications); + + var productClaims = BuildClaimsForVulnerability( + vulnerabilityId, + vulnerability, + productCatalog, + justifications, + detail, + unsupportedStatuses); + + claimsBuilder.AddRange(productClaims); + } + } + + return new CsafParseResult( + firstRelease, + lastRelease, + revision, + metadataBuilder.ToImmutable(), + claimsBuilder.ToImmutable(), + unsupportedStatuses.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray(), + unsupportedJustifications.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray(), + conflictingJustifications.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); + } + + private static IReadOnlyList BuildClaimsForVulnerability( + string vulnerabilityId, + JsonElement vulnerability, + IReadOnlyDictionary productCatalog, + ImmutableDictionary justifications, + string? detail, + ISet unsupportedStatuses) + { + if (!vulnerability.TryGetProperty("product_status", out var statusElement) || + statusElement.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var claims = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var statusProperty in statusElement.EnumerateObject()) + { + var status = MapStatus(statusProperty.Name, unsupportedStatuses); + if (status is null) + { + continue; + } + + if (statusProperty.Value.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var productIdElement in statusProperty.Value.EnumerateArray()) + { + var productId = productIdElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + var trimmedProductId = productId.Trim(); + var product = ResolveProduct(productCatalog, trimmedProductId); + justifications.TryGetValue(trimmedProductId, out var justificationInfo); + + UpdateClaim(claims, product, status.Value, statusProperty.Name, detail, justificationInfo); + } + } + + if (claims.Count == 0) + { + return Array.Empty(); + } + + return claims.Values + .Select(builder => new CsafClaimEntry( + vulnerabilityId, + builder.Product, + builder.Status, + builder.RawStatus, + builder.Detail, + builder.Justification, + builder.RawJustification)) + .ToArray(); + } + + private static void UpdateClaim( + IDictionary claims, + CsafProductInfo product, + VexClaimStatus status, + string rawStatus, + string? detail, + CsafJustificationInfo? justification) + { + if (!claims.TryGetValue(product.ProductId, out var existing) || + StatusPrecedence[status] > StatusPrecedence[existing.Status]) + { + claims[product.ProductId] = new CsafClaimEntryBuilder( + product, + status, + NormalizeRaw(rawStatus), + detail, + justification?.Normalized, + justification?.RawValue); + return; + } + + if (StatusPrecedence[status] < StatusPrecedence[existing.Status]) + { + return; + } + + var updated = existing; + + if (string.IsNullOrWhiteSpace(existing.RawStatus)) + { + updated = updated with { RawStatus = NormalizeRaw(rawStatus) }; + } + + if (existing.Detail is null && detail is not null) + { + updated = updated with { Detail = detail }; + } + + if (justification is not null) + { + if (existing.Justification is null && justification.Normalized is not null) + { + updated = updated with + { + Justification = justification.Normalized, + RawJustification = justification.RawValue + }; + } + else if (existing.Justification is null && + justification.Normalized is null && + string.IsNullOrWhiteSpace(existing.RawJustification) && + !string.IsNullOrWhiteSpace(justification.RawValue)) + { + updated = updated with { RawJustification = justification.RawValue }; + } + } + + claims[product.ProductId] = updated; + } + + private static CsafProductInfo ResolveProduct( + IReadOnlyDictionary catalog, + string productId) + { + if (catalog.TryGetValue(productId, out var product)) + { + return product; + } + + return new CsafProductInfo(productId, productId, null, null, null); + } + + private static string ResolveVulnerabilityId(JsonElement vulnerability) + { + var id = TryGetString(vulnerability, "cve") + ?? TryGetString(vulnerability, "id") + ?? TryGetString(vulnerability, "vuln_id"); + + return string.IsNullOrWhiteSpace(id) ? string.Empty : id.Trim(); + } + + private static string? ResolveDetail(JsonElement vulnerability) + { + var title = TryGetString(vulnerability, "title"); + if (!string.IsNullOrWhiteSpace(title)) + { + return title.Trim(); + } + + if (vulnerability.TryGetProperty("notes", out var notesElement) && + notesElement.ValueKind == JsonValueKind.Array) + { + foreach (var note in notesElement.EnumerateArray()) + { + if (note.ValueKind != JsonValueKind.Object) + { + continue; + } + + var category = TryGetString(note, "category"); + if (!string.IsNullOrWhiteSpace(category) && + !string.Equals(category, "description", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var text = TryGetString(note, "text"); + if (!string.IsNullOrWhiteSpace(text)) + { + return text.Trim(); + } + } + } + + return null; + } + + private static Dictionary CollectProducts(JsonElement root) + { + var products = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (!root.TryGetProperty("product_tree", out var productTree) || + productTree.ValueKind != JsonValueKind.Object) + { + return products; + } + + if (productTree.TryGetProperty("full_product_names", out var fullNames) && + fullNames.ValueKind == JsonValueKind.Array) + { + foreach (var productEntry in fullNames.EnumerateArray()) + { + var product = ParseProduct(productEntry, parentBranchName: null); + if (product is not null) + { + AddOrUpdate(product); + } + } + } + + if (productTree.TryGetProperty("branches", out var branches) && + branches.ValueKind == JsonValueKind.Array) + { + foreach (var branch in branches.EnumerateArray()) + { + VisitBranch(branch, parentBranchName: null); + } + } + + return products; + + void VisitBranch(JsonElement branch, string? parentBranchName) + { + if (branch.ValueKind != JsonValueKind.Object) + { + return; + } + + var branchName = TryGetString(branch, "name") ?? parentBranchName; + + if (branch.TryGetProperty("product", out var productElement)) + { + var product = ParseProduct(productElement, branchName); + if (product is not null) + { + AddOrUpdate(product); + } + } + + if (branch.TryGetProperty("branches", out var childBranches) && + childBranches.ValueKind == JsonValueKind.Array) + { + foreach (var childBranch in childBranches.EnumerateArray()) + { + VisitBranch(childBranch, branchName); + } + } + } + + void AddOrUpdate(CsafProductInfo product) + { + if (products.TryGetValue(product.ProductId, out var existing)) + { + products[product.ProductId] = MergeProducts(existing, product); + } + else + { + products[product.ProductId] = product; + } + } + + static CsafProductInfo MergeProducts(CsafProductInfo existing, CsafProductInfo incoming) + { + static string ChooseName(string incoming, string fallback) + => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; + + static string? ChooseOptional(string? incoming, string? fallback) + => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; + + return new CsafProductInfo( + existing.ProductId, + ChooseName(incoming.Name, existing.Name), + ChooseOptional(incoming.Version, existing.Version), + ChooseOptional(incoming.Purl, existing.Purl), + ChooseOptional(incoming.Cpe, existing.Cpe)); + } + } + + private static ImmutableDictionary> CollectProductGroups(JsonElement root) + { + if (!root.TryGetProperty("product_tree", out var productTree) || + productTree.ValueKind != JsonValueKind.Object || + !productTree.TryGetProperty("product_groups", out var groupsElement) || + groupsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableDictionary>.Empty; + } + + var groups = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var group in groupsElement.EnumerateArray()) + { + if (group.ValueKind != JsonValueKind.Object) + { + continue; + } + + var groupId = TryGetString(group, "group_id"); + if (string.IsNullOrWhiteSpace(groupId)) + { + continue; + } + + if (!group.TryGetProperty("product_ids", out var productIdsElement) || + productIdsElement.ValueKind != JsonValueKind.Array) + { + continue; + } + + var members = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var productIdElement in productIdsElement.EnumerateArray()) + { + var productId = productIdElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + members.Add(productId.Trim()); + } + + if (members.Count == 0) + { + continue; + } + + groups[groupId.Trim()] = members + .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + return groups.Count == 0 + ? ImmutableDictionary>.Empty + : groups.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + } + + private static ImmutableDictionary CollectJustifications( + JsonElement vulnerability, + IReadOnlyDictionary productCatalog, + ImmutableDictionary> productGroups, + ISet unsupportedJustifications, + ISet conflictingJustifications) + { + if (!vulnerability.TryGetProperty("flags", out var flagsElement) || + flagsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableDictionary.Empty; + } + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var flag in flagsElement.EnumerateArray()) + { + if (flag.ValueKind != JsonValueKind.Object) + { + continue; + } + + var label = TryGetString(flag, "label"); + if (string.IsNullOrWhiteSpace(label)) + { + continue; + } + + var rawLabel = NormalizeRaw(label); + var normalized = MapJustification(rawLabel, unsupportedJustifications); + + var targetIds = ExpandFlagProducts(flag, productGroups); + foreach (var productId in targetIds) + { + if (!productCatalog.ContainsKey(productId)) + { + continue; + } + + var info = new CsafJustificationInfo(rawLabel, normalized); + if (map.TryGetValue(productId, out var existing)) + { + if (existing.Normalized is null && normalized is not null) + { + map[productId] = info; + } + else if (existing.Normalized is not null && normalized is not null && existing.Normalized != normalized) + { + conflictingJustifications.Add(productId); + } + else if (existing.Normalized is null && + normalized is null && + string.IsNullOrWhiteSpace(existing.RawValue) && + !string.IsNullOrWhiteSpace(rawLabel)) + { + map[productId] = info; + } + } + else + { + map[productId] = info; + } + } + } + + return map.Count == 0 + ? ImmutableDictionary.Empty + : map.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + } + + private static IEnumerable ExpandFlagProducts( + JsonElement flag, + ImmutableDictionary> productGroups) + { + var productIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (flag.TryGetProperty("product_ids", out var productIdsElement) && + productIdsElement.ValueKind == JsonValueKind.Array) + { + foreach (var idElement in productIdsElement.EnumerateArray()) + { + var id = idElement.GetString(); + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + productIds.Add(id.Trim()); + } + } + + if (flag.TryGetProperty("group_ids", out var groupIdsElement) && + groupIdsElement.ValueKind == JsonValueKind.Array) + { + foreach (var groupIdElement in groupIdsElement.EnumerateArray()) + { + var groupId = groupIdElement.GetString(); + if (string.IsNullOrWhiteSpace(groupId)) + { + continue; + } + + if (productGroups.TryGetValue(groupId.Trim(), out var members)) + { + foreach (var member in members) + { + productIds.Add(member); + } + } + } + } + + return productIds; + } + + private static VexJustification? MapJustification(string justification, ISet unsupportedJustifications) + { + if (string.IsNullOrWhiteSpace(justification)) + { + return null; + } + + if (JustificationMap.TryGetValue(justification, out var mapped)) + { + return mapped; + } + + unsupportedJustifications.Add(justification); + return null; + } + + private static string NormalizeRaw(string value) + => string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim(); + + private static CsafProductInfo? ParseProduct(JsonElement element, string? parentBranchName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + JsonElement productElement = element; + if (!element.TryGetProperty("product_id", out var idElement) && + element.TryGetProperty("product", out var nestedProduct) && + nestedProduct.ValueKind == JsonValueKind.Object && + nestedProduct.TryGetProperty("product_id", out idElement)) + { + productElement = nestedProduct; + } + + var productId = idElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + return null; + } + + var name = TryGetString(productElement, "name") + ?? TryGetString(element, "name") + ?? parentBranchName + ?? productId; + + var version = TryGetString(productElement, "product_version") + ?? TryGetString(productElement, "version") + ?? TryGetString(element, "product_version"); + + string? cpe = null; + string? purl = null; + if (productElement.TryGetProperty("product_identification_helper", out var helper) && + helper.ValueKind == JsonValueKind.Object) + { + cpe = TryGetString(helper, "cpe"); + purl = TryGetString(helper, "purl"); + } + + return new CsafProductInfo(productId.Trim(), name.Trim(), version?.Trim(), purl?.Trim(), cpe?.Trim()); + } + + private static VexClaimStatus? MapStatus(string statusName, ISet unsupportedStatuses) + { + if (string.IsNullOrWhiteSpace(statusName)) + { + return null; + } + + var normalized = statusName.Trim(); + if (StatusMap.TryGetValue(normalized, out var mapped)) + { + return mapped; + } + + unsupportedStatuses.Add(normalized); + return null; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var dateElement)) + { + return null; + } + + var value = dateElement.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, out var parsed)) + { + return parsed; + } + + return null; + } + + private static string? TryGetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind == JsonValueKind.String ? property.GetString() : null; + } + + private static void AddIfPresent( + ImmutableDictionary.Builder builder, + JsonElement element, + string propertyName, + string metadataKey) + { + var value = TryGetString(element, propertyName); + if (!string.IsNullOrWhiteSpace(value)) + { + builder[metadataKey] = value.Trim(); + } + } + + private static void AddPublisherMetadata( + ImmutableDictionary.Builder builder, + JsonElement documentElement) + { + if (documentElement.ValueKind != JsonValueKind.Object || + !documentElement.TryGetProperty("publisher", out var publisher) || + publisher.ValueKind != JsonValueKind.Object) + { + return; + } + + AddIfPresent(builder, publisher, "name", "csaf.publisher.name"); + AddIfPresent(builder, publisher, "category", "csaf.publisher.category"); + } + + private readonly record struct CsafClaimEntryBuilder( + CsafProductInfo Product, + VexClaimStatus Status, + string RawStatus, + string? Detail, + VexJustification? Justification, + string? RawJustification); + } + + private sealed record CsafParseResult( + DateTimeOffset FirstRelease, + DateTimeOffset LastRelease, + string? Revision, + ImmutableDictionary Metadata, + ImmutableArray Claims, + ImmutableArray UnsupportedStatuses, + ImmutableArray UnsupportedJustifications, + ImmutableArray ConflictingJustifications); + + private sealed record CsafJustificationInfo( + string RawValue, + VexJustification? Normalized); + + private sealed record CsafClaimEntry( + string VulnerabilityId, + CsafProductInfo Product, + VexClaimStatus Status, + string RawStatus, + string? Detail, + VexJustification? Justification, + string? RawJustification); + + private sealed record CsafProductInfo( + string ProductId, + string Name, + string? Version, + string? Purl, + string? Cpe); +} diff --git a/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs b/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs index 907f85f8..e6811c97 100644 --- a/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs +++ b/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs @@ -1,116 +1,119 @@ -using System; -using System.Collections.Immutable; -using System.Linq; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using StellaOps.Plugin; -using StellaOps.Excititor.Core; -using StellaOps.Excititor.Storage.Mongo; - -namespace StellaOps.Excititor.Worker.Scheduling; - -internal sealed class DefaultVexProviderRunner : IVexProviderRunner -{ - private readonly IServiceProvider _serviceProvider; - private readonly PluginCatalog _pluginCatalog; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - - public DefaultVexProviderRunner( - IServiceProvider serviceProvider, - PluginCatalog pluginCatalog, - ILogger logger, - TimeProvider timeProvider) - { - _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); - _pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); - } - - public async ValueTask RunAsync(string providerId, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrWhiteSpace(providerId); - - using var scope = _serviceProvider.CreateScope(); - var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider); - var matched = availablePlugins.FirstOrDefault(plugin => - string.Equals(plugin.Name, providerId, StringComparison.OrdinalIgnoreCase)); - - if (matched is not null) - { - _logger.LogInformation( - "Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.", - matched.Name, - providerId); - } - else - { - _logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", providerId); - } - - var connectors = scope.ServiceProvider.GetServices(); - var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, providerId, StringComparison.OrdinalIgnoreCase)); - - if (connector is null) - { - _logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", providerId); - return; - } - - await ExecuteConnectorAsync(scope.ServiceProvider, connector, cancellationToken).ConfigureAwait(false); - } - - private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, CancellationToken cancellationToken) - { - var rawStore = scopeProvider.GetRequiredService(); - var claimStore = scopeProvider.GetRequiredService(); - var providerStore = scopeProvider.GetRequiredService(); - var normalizerRouter = scopeProvider.GetRequiredService(); - var signatureVerifier = scopeProvider.GetRequiredService(); - var sessionProvider = scopeProvider.GetRequiredService(); - var session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); - - var descriptor = connector switch - { - VexConnectorBase baseConnector => baseConnector.Descriptor, - _ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id) - }; - - var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false) - ?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind); - - await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false); - - await connector.ValidateAsync(VexConnectorSettings.Empty, cancellationToken).ConfigureAwait(false); - - var context = new VexConnectorContext( - Since: null, - Settings: VexConnectorSettings.Empty, - RawSink: rawStore, - SignatureVerifier: signatureVerifier, - Normalizers: normalizerRouter, - Services: scopeProvider); - - var documentCount = 0; - var claimCount = 0; - - await foreach (var document in connector.FetchAsync(context, cancellationToken)) - { - documentCount++; - - var batch = await normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false); - if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0) - { - claimCount += batch.Claims.Length; - await claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false); - } - } - - _logger.LogInformation( - "Connector {ConnectorId} persisted {DocumentCount} document(s) and {ClaimCount} claim(s) this run.", - connector.Id, - documentCount, - claimCount); - } -} +using System; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Plugin; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Worker.Scheduling; + +internal sealed class DefaultVexProviderRunner : IVexProviderRunner +{ + private readonly IServiceProvider _serviceProvider; + private readonly PluginCatalog _pluginCatalog; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public DefaultVexProviderRunner( + IServiceProvider serviceProvider, + PluginCatalog pluginCatalog, + ILogger logger, + TimeProvider timeProvider) + { + _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + _pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public async ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(schedule); + ArgumentException.ThrowIfNullOrWhiteSpace(schedule.ProviderId); + + using var scope = _serviceProvider.CreateScope(); + var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider); + var matched = availablePlugins.FirstOrDefault(plugin => + string.Equals(plugin.Name, schedule.ProviderId, StringComparison.OrdinalIgnoreCase)); + + if (matched is not null) + { + _logger.LogInformation( + "Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.", + matched.Name, + schedule.ProviderId); + } + else + { + _logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", schedule.ProviderId); + } + + var connectors = scope.ServiceProvider.GetServices(); + var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, schedule.ProviderId, StringComparison.OrdinalIgnoreCase)); + + if (connector is null) + { + _logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", schedule.ProviderId); + return; + } + + await ExecuteConnectorAsync(scope.ServiceProvider, connector, schedule.Settings, cancellationToken).ConfigureAwait(false); + } + + private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, VexConnectorSettings settings, CancellationToken cancellationToken) + { + var effectiveSettings = settings ?? VexConnectorSettings.Empty; + var rawStore = scopeProvider.GetRequiredService(); + var claimStore = scopeProvider.GetRequiredService(); + var providerStore = scopeProvider.GetRequiredService(); + var normalizerRouter = scopeProvider.GetRequiredService(); + var signatureVerifier = scopeProvider.GetRequiredService(); + var sessionProvider = scopeProvider.GetRequiredService(); + var session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + + var descriptor = connector switch + { + VexConnectorBase baseConnector => baseConnector.Descriptor, + _ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id) + }; + + var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false) + ?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind); + + await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false); + + await connector.ValidateAsync(effectiveSettings, cancellationToken).ConfigureAwait(false); + + var context = new VexConnectorContext( + Since: null, + Settings: effectiveSettings, + RawSink: rawStore, + SignatureVerifier: signatureVerifier, + Normalizers: normalizerRouter, + Services: scopeProvider); + + var documentCount = 0; + var claimCount = 0; + + await foreach (var document in connector.FetchAsync(context, cancellationToken)) + { + documentCount++; + + var batch = await normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false); + if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0) + { + claimCount += batch.Claims.Length; + await claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false); + } + } + + _logger.LogInformation( + "Connector {ConnectorId} persisted {DocumentCount} document(s) and {ClaimCount} claim(s) this run.", + connector.Id, + documentCount, + claimCount); + } +} diff --git a/src/StellaOps.Zastava.Core.Tests/Serialization/ZastavaCanonicalJsonSerializerTests.cs b/src/StellaOps.Zastava.Core.Tests/Serialization/ZastavaCanonicalJsonSerializerTests.cs index 6c520e6e..0530841a 100644 --- a/src/StellaOps.Zastava.Core.Tests/Serialization/ZastavaCanonicalJsonSerializerTests.cs +++ b/src/StellaOps.Zastava.Core.Tests/Serialization/ZastavaCanonicalJsonSerializerTests.cs @@ -1,204 +1,205 @@ -using System.Text; -using StellaOps.Zastava.Core.Contracts; -using StellaOps.Zastava.Core.Hashing; -using StellaOps.Zastava.Core.Serialization; - -namespace StellaOps.Zastava.Core.Tests.Serialization; - -public sealed class ZastavaCanonicalJsonSerializerTests -{ - [Fact] - public void Serialize_RuntimeEventEnvelope_ProducesDeterministicOrdering() - { - var runtimeEvent = new RuntimeEvent - { - EventId = "evt-123", - When = DateTimeOffset.Parse("2025-10-19T12:34:56Z"), - Kind = RuntimeEventKind.ContainerStart, - Tenant = "tenant-01", - Node = "node-a", - Runtime = new RuntimeEngine - { - Engine = "containerd", - Version = "1.7.19" - }, - Workload = new RuntimeWorkload - { - Platform = "kubernetes", - Namespace = "payments", - Pod = "api-7c9fbbd8b7-ktd84", - Container = "api", - ContainerId = "containerd://abc", - ImageRef = "ghcr.io/acme/api@sha256:abcd", - Owner = new RuntimeWorkloadOwner - { - Kind = "Deployment", - Name = "api" - } - }, - Process = new RuntimeProcess - { - Pid = 12345, - Entrypoint = new[] { "/entrypoint.sh", "--serve" }, - EntryTrace = new[] - { - new RuntimeEntryTrace - { - File = "/entrypoint.sh", - Line = 3, - Op = "exec", - Target = "/usr/bin/python3" - } - } - }, - LoadedLibraries = new[] - { - new RuntimeLoadedLibrary - { - Path = "/lib/x86_64-linux-gnu/libssl.so.3", - Inode = 123456, - Sha256 = "abc123" - } - }, - Posture = new RuntimePosture - { - ImageSigned = true, - SbomReferrer = "present", - Attestation = new RuntimeAttestation - { - Uuid = "rekor-uuid", - Verified = true - } - }, - Delta = new RuntimeDelta - { - BaselineImageDigest = "sha256:abcd", - ChangedFiles = new[] { "/opt/app/server.py" }, - NewBinaries = new[] - { - new RuntimeNewBinary - { - Path = "/usr/local/bin/helper", - Sha256 = "def456" - } - } - }, - Evidence = new[] - { - new RuntimeEvidence - { - Signal = "procfs.maps", - Value = "/lib/.../libssl.so.3@0x7f..." - } - }, - Annotations = new Dictionary - { - ["source"] = "unit-test" - } - }; - - var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); - var json = ZastavaCanonicalJsonSerializer.Serialize(envelope); - - var expectedOrder = new[] - { - "\"schemaVersion\"", - "\"event\"", - "\"eventId\"", - "\"when\"", - "\"kind\"", - "\"tenant\"", - "\"node\"", - "\"runtime\"", - "\"engine\"", - "\"version\"", - "\"workload\"", - "\"platform\"", - "\"namespace\"", - "\"pod\"", - "\"container\"", - "\"containerId\"", - "\"imageRef\"", - "\"owner\"", - "\"kind\"", - "\"name\"", - "\"process\"", - "\"pid\"", - "\"entrypoint\"", - "\"entryTrace\"", - "\"loadedLibs\"", - "\"posture\"", - "\"imageSigned\"", - "\"sbomReferrer\"", - "\"attestation\"", - "\"uuid\"", - "\"verified\"", - "\"delta\"", - "\"baselineImageDigest\"", - "\"changedFiles\"", - "\"newBinaries\"", - "\"path\"", - "\"sha256\"", - "\"evidence\"", - "\"signal\"", - "\"value\"", - "\"annotations\"", - "\"source\"" - }; - - var cursor = -1; - foreach (var token in expectedOrder) - { - var position = json.IndexOf(token, cursor + 1, StringComparison.Ordinal); - Assert.True(position > cursor, $"Property token {token} not found in the expected order."); - cursor = position; - } - - Assert.DoesNotContain(" ", json, StringComparison.Ordinal); - Assert.StartsWith("{\"schemaVersion\"", json, StringComparison.Ordinal); - Assert.EndsWith("}}", json, StringComparison.Ordinal); - } - - [Fact] - public void ComputeMultihash_ProducesStableBase64UrlDigest() - { - var decision = AdmissionDecisionEnvelope.Create( - new AdmissionDecision - { - AdmissionId = "admission-123", - Namespace = "payments", - PodSpecDigest = "sha256:deadbeef", - Images = new[] - { - new AdmissionImageVerdict - { - Name = "ghcr.io/acme/api:1.2.3", - Resolved = "ghcr.io/acme/api@sha256:abcd", - Signed = true, - HasSbomReferrers = true, - PolicyVerdict = PolicyVerdict.Pass, - Reasons = Array.Empty(), - Rekor = new AdmissionRekorEvidence - { - Uuid = "xyz", - Verified = true - } - } - }, - Decision = AdmissionDecisionOutcome.Allow, - TtlSeconds = 300 - }, - ZastavaContractVersions.AdmissionDecision); - - var canonicalJson = ZastavaCanonicalJsonSerializer.Serialize(decision); - var expectedDigestBytes = SHA256.HashData(Encoding.UTF8.GetBytes(canonicalJson)); - var expected = $"sha256-{Convert.ToBase64String(expectedDigestBytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}"; - - var hash = ZastavaHashing.ComputeMultihash(decision); - - Assert.Equal(expected, hash); - - var sha512 = ZastavaHashing.ComputeMultihash(Encoding.UTF8.GetBytes(canonicalJson), "sha512"); - Assert.StartsWith("sha512-", sha512, StringComparison.Ordinal); - } -} +using System.Text; +using System.Security.Cryptography; +using StellaOps.Zastava.Core.Contracts; +using StellaOps.Zastava.Core.Hashing; +using StellaOps.Zastava.Core.Serialization; + +namespace StellaOps.Zastava.Core.Tests.Serialization; + +public sealed class ZastavaCanonicalJsonSerializerTests +{ + [Fact] + public void Serialize_RuntimeEventEnvelope_ProducesDeterministicOrdering() + { + var runtimeEvent = new RuntimeEvent + { + EventId = "evt-123", + When = DateTimeOffset.Parse("2025-10-19T12:34:56Z"), + Kind = RuntimeEventKind.ContainerStart, + Tenant = "tenant-01", + Node = "node-a", + Runtime = new RuntimeEngine + { + Engine = "containerd", + Version = "1.7.19" + }, + Workload = new RuntimeWorkload + { + Platform = "kubernetes", + Namespace = "payments", + Pod = "api-7c9fbbd8b7-ktd84", + Container = "api", + ContainerId = "containerd://abc", + ImageRef = "ghcr.io/acme/api@sha256:abcd", + Owner = new RuntimeWorkloadOwner + { + Kind = "Deployment", + Name = "api" + } + }, + Process = new RuntimeProcess + { + Pid = 12345, + Entrypoint = new[] { "/entrypoint.sh", "--serve" }, + EntryTrace = new[] + { + new RuntimeEntryTrace + { + File = "/entrypoint.sh", + Line = 3, + Op = "exec", + Target = "/usr/bin/python3" + } + } + }, + LoadedLibraries = new[] + { + new RuntimeLoadedLibrary + { + Path = "/lib/x86_64-linux-gnu/libssl.so.3", + Inode = 123456, + Sha256 = "abc123" + } + }, + Posture = new RuntimePosture + { + ImageSigned = true, + SbomReferrer = "present", + Attestation = new RuntimeAttestation + { + Uuid = "rekor-uuid", + Verified = true + } + }, + Delta = new RuntimeDelta + { + BaselineImageDigest = "sha256:abcd", + ChangedFiles = new[] { "/opt/app/server.py" }, + NewBinaries = new[] + { + new RuntimeNewBinary + { + Path = "/usr/local/bin/helper", + Sha256 = "def456" + } + } + }, + Evidence = new[] + { + new RuntimeEvidence + { + Signal = "procfs.maps", + Value = "/lib/.../libssl.so.3@0x7f..." + } + }, + Annotations = new Dictionary + { + ["source"] = "unit-test" + } + }; + + var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); + var json = ZastavaCanonicalJsonSerializer.Serialize(envelope); + + var expectedOrder = new[] + { + "\"schemaVersion\"", + "\"event\"", + "\"eventId\"", + "\"when\"", + "\"kind\"", + "\"tenant\"", + "\"node\"", + "\"runtime\"", + "\"engine\"", + "\"version\"", + "\"workload\"", + "\"platform\"", + "\"namespace\"", + "\"pod\"", + "\"container\"", + "\"containerId\"", + "\"imageRef\"", + "\"owner\"", + "\"kind\"", + "\"name\"", + "\"process\"", + "\"pid\"", + "\"entrypoint\"", + "\"entryTrace\"", + "\"loadedLibs\"", + "\"posture\"", + "\"imageSigned\"", + "\"sbomReferrer\"", + "\"attestation\"", + "\"uuid\"", + "\"verified\"", + "\"delta\"", + "\"baselineImageDigest\"", + "\"changedFiles\"", + "\"newBinaries\"", + "\"path\"", + "\"sha256\"", + "\"evidence\"", + "\"signal\"", + "\"value\"", + "\"annotations\"", + "\"source\"" + }; + + var cursor = -1; + foreach (var token in expectedOrder) + { + var position = json.IndexOf(token, cursor + 1, StringComparison.Ordinal); + Assert.True(position > cursor, $"Property token {token} not found in the expected order."); + cursor = position; + } + + Assert.DoesNotContain(" ", json, StringComparison.Ordinal); + Assert.StartsWith("{\"schemaVersion\"", json, StringComparison.Ordinal); + Assert.EndsWith("}}", json, StringComparison.Ordinal); + } + + [Fact] + public void ComputeMultihash_ProducesStableBase64UrlDigest() + { + var decision = AdmissionDecisionEnvelope.Create( + new AdmissionDecision + { + AdmissionId = "admission-123", + Namespace = "payments", + PodSpecDigest = "sha256:deadbeef", + Images = new[] + { + new AdmissionImageVerdict + { + Name = "ghcr.io/acme/api:1.2.3", + Resolved = "ghcr.io/acme/api@sha256:abcd", + Signed = true, + HasSbomReferrers = true, + PolicyVerdict = PolicyVerdict.Pass, + Reasons = Array.Empty(), + Rekor = new AdmissionRekorEvidence + { + Uuid = "xyz", + Verified = true + } + } + }, + Decision = AdmissionDecisionOutcome.Allow, + TtlSeconds = 300 + }, + ZastavaContractVersions.AdmissionDecision); + + var canonicalJson = ZastavaCanonicalJsonSerializer.Serialize(decision); + var expectedDigestBytes = SHA256.HashData(Encoding.UTF8.GetBytes(canonicalJson)); + var expected = $"sha256-{Convert.ToBase64String(expectedDigestBytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}"; + + var hash = ZastavaHashing.ComputeMultihash(decision); + + Assert.Equal(expected, hash); + + var sha512 = ZastavaHashing.ComputeMultihash(Encoding.UTF8.GetBytes(canonicalJson), "sha512"); + Assert.StartsWith("sha512-", sha512, StringComparison.Ordinal); + } +} diff --git a/src/StellaOps.Zastava.Core/Contracts/ZastavaContractVersions.cs b/src/StellaOps.Zastava.Core/Contracts/ZastavaContractVersions.cs index 1751d2e0..0decf2f0 100644 --- a/src/StellaOps.Zastava.Core/Contracts/ZastavaContractVersions.cs +++ b/src/StellaOps.Zastava.Core/Contracts/ZastavaContractVersions.cs @@ -104,8 +104,8 @@ public static class ZastavaContractVersions /// /// Canonical string representation (schema@vMajor.Minor). /// - public override string ToString() - => $"{Schema}@v{Version.ToString(2, CultureInfo.InvariantCulture)}"; + public override string ToString() + => $"{Schema}@v{Version.ToString(2)}"; /// /// Determines whether a remote contract is compatible with the local definition. diff --git a/src/StellaOps.Zastava.Core/Serialization/ZastavaCanonicalJsonSerializer.cs b/src/StellaOps.Zastava.Core/Serialization/ZastavaCanonicalJsonSerializer.cs index e4219303..76f4b297 100644 --- a/src/StellaOps.Zastava.Core/Serialization/ZastavaCanonicalJsonSerializer.cs +++ b/src/StellaOps.Zastava.Core/Serialization/ZastavaCanonicalJsonSerializer.cs @@ -1,4 +1,13 @@ -namespace StellaOps.Zastava.Core.Serialization; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using StellaOps.Zastava.Core.Contracts; + +namespace StellaOps.Zastava.Core.Serialization; /// /// Deterministic serializer used for runtime/admission contracts.