Restore vendor connector internals and configure offline packages
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

This commit is contained in:
root
2025-10-20 14:50:58 +03:00
parent 09b6a28172
commit 44ad31591c
59 changed files with 7512 additions and 3797 deletions

17
NuGet.config Normal file
View File

@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<clear />
<add key="local" value="local-nuget" />
<add key="mirror" value="https://mirrors.ablera.dev/nuget/nuget-mirror/v3/index.json" />
</packageSources>
<packageSourceMapping>
<packageSource key="local">
<package pattern="Mongo2Go" />
<package pattern="Microsoft.Extensions.Http.Polly" />
</packageSource>
<packageSource key="mirror">
<package pattern="*" />
</packageSource>
</packageSourceMapping>
</configuration>

Binary file not shown.

View File

@@ -64,12 +64,13 @@ public class StandardClientProvisioningStoreTests
var result = await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None); var result = await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None);
Assert.True(result.Succeeded); Assert.True(result.Succeeded);
var document = Assert.Contains("signer", store.Documents); Assert.True(store.Documents.TryGetValue("signer", out var document));
Assert.Equal("attestor signer", document.Value.Properties[AuthorityClientMetadataKeys.Audiences]); Assert.NotNull(document);
Assert.Equal("attestor signer", document!.Properties[AuthorityClientMetadataKeys.Audiences]);
var descriptor = await provisioning.FindByClientIdAsync("signer", CancellationToken.None); var descriptor = await provisioning.FindByClientIdAsync("signer", CancellationToken.None);
Assert.NotNull(descriptor); Assert.NotNull(descriptor);
Assert.Equal(new[] { "attestor", "signer" }, descriptor!.Audiences.OrderBy(value => value, StringComparer.Ordinal)); Assert.Equal(new[] { "attestor", "signer" }, descriptor!.AllowedAudiences.OrderBy(value => value, StringComparer.Ordinal));
} }
[Fact] [Fact]
@@ -101,8 +102,9 @@ public class StandardClientProvisioningStoreTests
await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None); await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None);
var document = Assert.Contains("mtls-client", store.Documents).Value; Assert.True(store.Documents.TryGetValue("mtls-client", out var document));
var binding = Assert.Single(document.CertificateBindings); Assert.NotNull(document);
var binding = Assert.Single(document!.CertificateBindings);
Assert.Equal("AABBCCDD", binding.Thumbprint); Assert.Equal("AABBCCDD", binding.Thumbprint);
Assert.Equal("01ff", binding.SerialNumber); Assert.Equal("01ff", binding.SerialNumber);
Assert.Equal("CN=mtls-client", binding.Subject); Assert.Equal("CN=mtls-client", binding.Subject);

View File

@@ -54,11 +54,11 @@ internal sealed record BootstrapClientRequest
public IReadOnlyCollection<BootstrapClientCertificateBinding>? CertificateBindings { get; init; } public IReadOnlyCollection<BootstrapClientCertificateBinding>? CertificateBindings { get; init; }
} }
internal sealed record BootstrapInviteRequest internal sealed record BootstrapInviteRequest
{ {
public string Type { get; init; } = BootstrapInviteTypes.User; public string Type { get; init; } = BootstrapInviteTypes.User;
public string? Token { get; init; } public string? Token { get; init; }
public string? Provider { get; init; } public string? Provider { get; init; }
@@ -91,31 +91,8 @@ internal sealed record BootstrapClientCertificateBinding
public string? Label { get; init; } public string? Label { get; init; }
} }
internal static class BootstrapInviteTypes internal static class BootstrapInviteTypes
{ {
public const string User = "user"; public const string User = "user";
public const string Client = "client"; public const string Client = "client";
} }
internal sealed record BootstrapInviteRequest
{
public string Type { get; init; } = BootstrapInviteTypes.User;
public string? Token { get; init; }
public string? Provider { get; init; }
public string? Target { get; init; }
public DateTimeOffset? ExpiresAt { get; init; }
public string? IssuedBy { get; init; }
public IReadOnlyDictionary<string, string?>? Metadata { get; init; }
}
internal static class BootstrapInviteTypes
{
public const string User = "user";
public const string Client = "client";
}

View File

@@ -0,0 +1,104 @@
using System.Net;
namespace StellaOps.Concelier.Connector.CertBund.Configuration;
public sealed class CertBundOptions
{
public const string HttpClientName = "concelier.source.certbund";
/// <summary>
/// RSS feed providing the latest CERT-Bund advisories.
/// </summary>
public Uri FeedUri { get; set; } = new("https://wid.cert-bund.de/content/public/securityAdvisory/rss");
/// <summary>
/// Portal endpoint used to bootstrap session cookies (required for the SPA JSON API).
/// </summary>
public Uri PortalBootstrapUri { get; set; } = new("https://wid.cert-bund.de/portal/");
/// <summary>
/// Detail API endpoint template; advisory identifier is appended as the <c>name</c> query parameter.
/// </summary>
public Uri DetailApiUri { get; set; } = new("https://wid.cert-bund.de/portal/api/securityadvisory");
/// <summary>
/// Optional timeout override for feed/detail requests.
/// </summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Delay applied between successive detail fetches to respect upstream politeness.
/// </summary>
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
/// <summary>
/// Backoff recorded in source state when a fetch attempt fails.
/// </summary>
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Maximum number of advisories to enqueue per fetch iteration.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 50;
/// <summary>
/// Maximum number of advisory identifiers remembered to prevent re-processing.
/// </summary>
public int MaxKnownAdvisories { get; set; } = 512;
public void Validate()
{
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund feed URI must be an absolute URI.");
}
if (PortalBootstrapUri is null || !PortalBootstrapUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund portal bootstrap URI must be an absolute URI.");
}
if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CERT-Bund detail API URI must be an absolute URI.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (MaxKnownAdvisories <= 0)
{
throw new InvalidOperationException($"{nameof(MaxKnownAdvisories)} must be greater than zero.");
}
}
public Uri BuildDetailUri(string advisoryId)
{
if (string.IsNullOrWhiteSpace(advisoryId))
{
throw new ArgumentException("Advisory identifier must be provided.", nameof(advisoryId));
}
var builder = new UriBuilder(DetailApiUri);
var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&";
builder.Query = $"{queryPrefix}name={Uri.EscapeDataString(advisoryId)}";
return builder.Uri;
}
}

View File

@@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
public sealed record CertBundAdvisoryDto
{
[JsonPropertyName("advisoryId")]
public string AdvisoryId { get; init; } = string.Empty;
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("contentHtml")]
public string ContentHtml { get; init; } = string.Empty;
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("language")]
public string Language { get; init; } = "de";
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("portalUri")]
public Uri PortalUri { get; init; } = new("https://wid.cert-bund.de/");
[JsonPropertyName("detailUri")]
public Uri DetailUri { get; init; } = new("https://wid.cert-bund.de/");
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
[JsonPropertyName("products")]
public IReadOnlyList<CertBundProductDto> Products { get; init; } = Array.Empty<CertBundProductDto>();
[JsonPropertyName("references")]
public IReadOnlyList<CertBundReferenceDto> References { get; init; } = Array.Empty<CertBundReferenceDto>();
}
public sealed record CertBundProductDto
{
[JsonPropertyName("vendor")]
public string? Vendor { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("versions")]
public string? Versions { get; init; }
}
public sealed record CertBundReferenceDto
{
[JsonPropertyName("url")]
public string Url { get; init; } = string.Empty;
[JsonPropertyName("label")]
public string? Label { get; init; }
}

View File

@@ -0,0 +1,118 @@
using System;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
internal sealed record CertBundCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
IReadOnlyCollection<string> KnownAdvisories,
DateTimeOffset? LastPublished,
DateTimeOffset? LastFetchAt)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
private static readonly IReadOnlyCollection<string> EmptyStrings = Array.Empty<string>();
public static CertBundCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null);
public CertBundCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = Distinct(documents) };
public CertBundCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = Distinct(mappings) };
public CertBundCursor WithKnownAdvisories(IEnumerable<string> advisories)
=> this with { KnownAdvisories = advisories?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings };
public CertBundCursor WithLastPublished(DateTimeOffset? published)
=> this with { LastPublished = published };
public CertBundCursor WithLastFetch(DateTimeOffset? timestamp)
=> this with { LastFetchAt = timestamp };
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
["knownAdvisories"] = new BsonArray(KnownAdvisories),
};
if (LastPublished.HasValue)
{
document["lastPublished"] = LastPublished.Value.UtcDateTime;
}
if (LastFetchAt.HasValue)
{
document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
}
return document;
}
public static CertBundCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var knownAdvisories = ReadStringArray(document, "knownAdvisories");
var lastPublished = document.TryGetValue("lastPublished", out var publishedValue)
? ParseDate(publishedValue)
: null;
var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue)
? ParseDate(fetchValue)
: null;
return new CertBundCursor(pendingDocuments, pendingMappings, knownAdvisories, lastPublished, lastFetch);
}
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuids;
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuids;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var id))
{
items.Add(id);
}
}
return items;
}
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyStrings;
}
return array.Select(element => element?.ToString() ?? string.Empty)
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Concelier.Connector.Common.Html;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
public sealed class CertBundDetailParser
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly HtmlContentSanitizer _sanitizer;
public CertBundDetailParser(HtmlContentSanitizer sanitizer)
=> _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
public CertBundAdvisoryDto Parse(Uri detailUri, Uri portalUri, byte[] payload)
{
var detail = JsonSerializer.Deserialize<CertBundDetailResponse>(payload, SerializerOptions)
?? throw new InvalidOperationException("CERT-Bund detail payload deserialized to null.");
var advisoryId = detail.Name ?? throw new InvalidOperationException("CERT-Bund detail missing advisory name.");
var contentHtml = _sanitizer.Sanitize(detail.Description ?? string.Empty, portalUri);
return new CertBundAdvisoryDto
{
AdvisoryId = advisoryId,
Title = detail.Title ?? advisoryId,
Summary = detail.Summary,
ContentHtml = contentHtml,
Severity = detail.Severity,
Language = string.IsNullOrWhiteSpace(detail.Language) ? "de" : detail.Language!,
Published = detail.Published,
Modified = detail.Updated ?? detail.Published,
PortalUri = portalUri,
DetailUri = detailUri,
CveIds = detail.CveIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id!.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>(),
References = MapReferences(detail.References),
Products = MapProducts(detail.Products),
};
}
private static IReadOnlyList<CertBundReferenceDto> MapReferences(CertBundDetailReference[]? references)
{
if (references is null || references.Length == 0)
{
return Array.Empty<CertBundReferenceDto>();
}
return references
.Where(static reference => !string.IsNullOrWhiteSpace(reference.Url))
.Select(reference => new CertBundReferenceDto
{
Url = reference.Url!,
Label = reference.Label,
})
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<CertBundProductDto> MapProducts(CertBundDetailProduct[]? products)
{
if (products is null || products.Length == 0)
{
return Array.Empty<CertBundProductDto>();
}
return products
.Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name))
.Select(product => new CertBundProductDto
{
Vendor = product.Vendor,
Name = product.Name,
Versions = product.Versions,
})
.ToArray();
}
}

View File

@@ -0,0 +1,60 @@
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
internal sealed record CertBundDetailResponse
{
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("language")]
public string? Language { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("updated")]
public DateTimeOffset? Updated { get; init; }
[JsonPropertyName("cveIds")]
public string[]? CveIds { get; init; }
[JsonPropertyName("references")]
public CertBundDetailReference[]? References { get; init; }
[JsonPropertyName("products")]
public CertBundDetailProduct[]? Products { get; init; }
}
internal sealed record CertBundDetailReference
{
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("label")]
public string? Label { get; init; }
}
internal sealed record CertBundDetailProduct
{
[JsonPropertyName("vendor")]
public string? Vendor { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("versions")]
public string? Versions { get; init; }
}

View File

@@ -0,0 +1,191 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
/// <summary>
/// Emits OpenTelemetry counters and histograms for the CERT-Bund connector.
/// </summary>
public sealed class CertBundDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertBund";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _feedFetchAttempts;
private readonly Counter<long> _feedFetchSuccess;
private readonly Counter<long> _feedFetchFailures;
private readonly Histogram<long> _feedItemCount;
private readonly Histogram<long> _feedEnqueuedCount;
private readonly Histogram<double> _feedCoverageDays;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchNotModified;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseProductCount;
private readonly Histogram<long> _parseCveCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapPackageCount;
private readonly Histogram<long> _mapAliasCount;
public CertBundDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_feedFetchAttempts = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.attempts",
unit: "operations",
description: "Number of RSS feed load attempts.");
_feedFetchSuccess = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.success",
unit: "operations",
description: "Number of successful RSS feed loads.");
_feedFetchFailures = _meter.CreateCounter<long>(
name: "certbund.feed.fetch.failures",
unit: "operations",
description: "Number of RSS feed load failures.");
_feedItemCount = _meter.CreateHistogram<long>(
name: "certbund.feed.items.count",
unit: "items",
description: "Distribution of RSS item counts per fetch.");
_feedEnqueuedCount = _meter.CreateHistogram<long>(
name: "certbund.feed.enqueued.count",
unit: "documents",
description: "Distribution of advisory documents enqueued per fetch.");
_feedCoverageDays = _meter.CreateHistogram<double>(
name: "certbund.feed.coverage.days",
unit: "days",
description: "Coverage window in days between fetch time and the oldest published advisory in the feed.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.attempts",
unit: "operations",
description: "Number of detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.success",
unit: "operations",
description: "Number of detail fetches that persisted a document.");
_detailFetchNotModified = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.not_modified",
unit: "operations",
description: "Number of detail fetches returning HTTP 304.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certbund.detail.fetch.failures",
unit: "operations",
description: "Number of detail fetches that failed.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certbund.parse.success",
unit: "documents",
description: "Number of documents parsed into CERT-Bund DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certbund.parse.failures",
unit: "documents",
description: "Number of documents that failed to parse.");
_parseProductCount = _meter.CreateHistogram<long>(
name: "certbund.parse.products.count",
unit: "products",
description: "Distribution of product entries captured per advisory.");
_parseCveCount = _meter.CreateHistogram<long>(
name: "certbund.parse.cve.count",
unit: "aliases",
description: "Distribution of CVE identifiers captured per advisory.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certbund.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certbund.map.failures",
unit: "advisories",
description: "Number of mapping failures.");
_mapPackageCount = _meter.CreateHistogram<long>(
name: "certbund.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per advisory.");
_mapAliasCount = _meter.CreateHistogram<long>(
name: "certbund.map.aliases.count",
unit: "aliases",
description: "Distribution of alias counts per advisory.");
}
public void FeedFetchAttempt() => _feedFetchAttempts.Add(1);
public void FeedFetchSuccess(int itemCount)
{
_feedFetchSuccess.Add(1);
if (itemCount >= 0)
{
_feedItemCount.Record(itemCount);
}
}
public void FeedFetchFailure(string reason = "error")
=> _feedFetchFailures.Add(1, ReasonTag(reason));
public void RecordFeedCoverage(double? coverageDays)
{
if (coverageDays is { } days && days >= 0)
{
_feedCoverageDays.Record(days);
}
}
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
public void DetailFetchFailure(string reason = "error")
=> _detailFetchFailures.Add(1, ReasonTag(reason));
public void DetailFetchEnqueued(int count)
{
if (count >= 0)
{
_feedEnqueuedCount.Record(count);
}
}
public void ParseSuccess(int productCount, int cveCount)
{
_parseSuccess.Add(1);
if (productCount >= 0)
{
_parseProductCount.Record(productCount);
}
if (cveCount >= 0)
{
_parseCveCount.Record(cveCount);
}
}
public void ParseFailure(string reason = "error")
=> _parseFailures.Add(1, ReasonTag(reason));
public void MapSuccess(int affectedPackages, int aliasCount)
{
_mapSuccess.Add(1);
if (affectedPackages >= 0)
{
_mapPackageCount.Record(affectedPackages);
}
if (aliasCount >= 0)
{
_mapAliasCount.Record(aliasCount);
}
}
public void MapFailure(string reason = "error")
=> _mapFailures.Add(1, ReasonTag(reason));
private static KeyValuePair<string, object?> ReasonTag(string reason)
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
internal static class CertBundDocumentMetadata
{
public static Dictionary<string, string> CreateMetadata(CertBundFeedItem item)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["certbund.advisoryId"] = item.AdvisoryId,
["certbund.portalUri"] = item.PortalUri.ToString(),
["certbund.published"] = item.Published.ToString("O"),
};
if (!string.IsNullOrWhiteSpace(item.Category))
{
metadata["certbund.category"] = item.Category!;
}
if (!string.IsNullOrWhiteSpace(item.Title))
{
metadata["certbund.title"] = item.Title!;
}
return metadata;
}
}

View File

@@ -0,0 +1,143 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertBund.Configuration;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
public sealed class CertBundFeedClient
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly CertBundOptions _options;
private readonly ILogger<CertBundFeedClient> _logger;
private readonly SemaphoreSlim _bootstrapSemaphore = new(1, 1);
private volatile bool _bootstrapped;
public CertBundFeedClient(
IHttpClientFactory httpClientFactory,
IOptions<CertBundOptions> options,
ILogger<CertBundFeedClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<CertBundFeedItem>> LoadAsync(CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CertBundOptions.HttpClientName);
await EnsureSessionAsync(client, cancellationToken).ConfigureAwait(false);
using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri);
request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8");
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var document = XDocument.Load(stream);
var items = new List<CertBundFeedItem>();
foreach (var element in document.Descendants("item"))
{
cancellationToken.ThrowIfCancellationRequested();
var linkValue = element.Element("link")?.Value?.Trim();
if (string.IsNullOrWhiteSpace(linkValue) || !Uri.TryCreate(linkValue, UriKind.Absolute, out var portalUri))
{
continue;
}
var advisoryId = TryExtractNameParameter(portalUri);
if (string.IsNullOrWhiteSpace(advisoryId))
{
continue;
}
var detailUri = _options.BuildDetailUri(advisoryId);
var pubDateText = element.Element("pubDate")?.Value;
var published = ParseDate(pubDateText);
var title = element.Element("title")?.Value?.Trim();
var category = element.Element("category")?.Value?.Trim();
items.Add(new CertBundFeedItem(advisoryId, detailUri, portalUri, published, title, category));
}
return items;
}
private async Task EnsureSessionAsync(HttpClient client, CancellationToken cancellationToken)
{
if (_bootstrapped)
{
return;
}
await _bootstrapSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_bootstrapped)
{
return;
}
using var request = new HttpRequestMessage(HttpMethod.Get, _options.PortalBootstrapUri);
request.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
_bootstrapped = true;
}
finally
{
_bootstrapSemaphore.Release();
}
}
private static string? TryExtractNameParameter(Uri portalUri)
{
if (portalUri is null)
{
return null;
}
var query = portalUri.Query;
if (string.IsNullOrEmpty(query))
{
return null;
}
var trimmed = query.TrimStart('?');
foreach (var pair in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries))
{
var separatorIndex = pair.IndexOf('=');
if (separatorIndex <= 0)
{
continue;
}
var key = pair[..separatorIndex].Trim();
if (!key.Equals("name", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var value = pair[(separatorIndex + 1)..];
return Uri.UnescapeDataString(value);
}
return null;
}
private static DateTimeOffset ParseDate(string? value)
=> DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
? parsed
: DateTimeOffset.UtcNow;
}

View File

@@ -0,0 +1,11 @@
namespace StellaOps.Concelier.Connector.CertBund.Internal;
using System;
public sealed record CertBundFeedItem(
string AdvisoryId,
Uri DetailUri,
Uri PortalUri,
DateTimeOffset Published,
string? Title,
string? Category);

View File

@@ -0,0 +1,168 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Documents;
namespace StellaOps.Concelier.Connector.CertBund.Internal;
internal static class CertBundMapper
{
public static Advisory Map(CertBundAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var packages = BuildPackages(dto, recordedAt);
var provenance = new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"advisory",
dto.AdvisoryId,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory });
return new Advisory(
advisoryKey: dto.AdvisoryId,
title: dto.Title,
summary: dto.Summary,
language: dto.Language?.ToLowerInvariant() ?? "de",
published: dto.Published,
modified: dto.Modified,
severity: MapSeverity(dto.Severity),
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: packages,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: new[] { provenance });
}
private static IReadOnlyList<string> BuildAliases(CertBundAdvisoryDto dto)
{
var aliases = new List<string>(capacity: 4) { dto.AdvisoryId };
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>
{
new(dto.DetailUri.ToString(), "details", "cert-bund", null, new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"reference",
dto.DetailUri.ToString(),
recordedAt,
new[] { ProvenanceFieldMasks.References }))
};
foreach (var reference in dto.References)
{
if (string.IsNullOrWhiteSpace(reference.Url))
{
continue;
}
references.Add(new AdvisoryReference(
reference.Url,
kind: "reference",
sourceTag: "cert-bund",
summary: reference.Label,
provenance: new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"reference",
reference.Url,
recordedAt,
new[] { ProvenanceFieldMasks.References })));
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<AffectedPackage> BuildPackages(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
var vendor = Validation.TrimToNull(product.Vendor) ?? "Unspecified";
var name = Validation.TrimToNull(product.Name);
var identifier = name is null ? vendor : $"{vendor} {name}";
var provenance = new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var ranges = string.IsNullOrWhiteSpace(product.Versions)
? Array.Empty<AffectedVersionRange>()
: new[]
{
new AffectedVersionRange(
rangeKind: "string",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: product.Versions,
provenance: new AdvisoryProvenance(
CertBundConnectorPlugin.SourceName,
"package-range",
product.Versions,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges }))
};
packages.Add(new AffectedPackage(
AffectedPackageTypes.Vendor,
identifier,
platform: null,
versionRanges: ranges,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static string? MapSeverity(string? severity)
{
if (string.IsNullOrWhiteSpace(severity))
{
return null;
}
return severity.ToLowerInvariant() switch
{
"hoch" or "high" => "high",
"mittel" or "medium" => "medium",
"gering" or "low" => "low",
_ => severity.ToLowerInvariant(),
};
}
}

View File

@@ -1,162 +1,187 @@
using MongoDB.Bson; using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common.Cursors; using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal; namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal sealed record CertCcCursor( internal sealed record CertCcCursor(
TimeWindowCursorState SummaryState, TimeWindowCursorState SummaryState,
IReadOnlyCollection<Guid> PendingSummaries, IReadOnlyCollection<Guid> PendingSummaries,
IReadOnlyCollection<string> PendingNotes, IReadOnlyCollection<string> PendingNotes,
IReadOnlyCollection<Guid> PendingDocuments, IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings, IReadOnlyCollection<Guid> PendingMappings,
DateTimeOffset? LastRun) DateTimeOffset? LastRun)
{ {
private static readonly Guid[] EmptyGuidArray = Array.Empty<Guid>(); private static readonly Guid[] EmptyGuidArray = Array.Empty<Guid>();
private static readonly string[] EmptyStringArray = Array.Empty<string>(); private static readonly string[] EmptyStringArray = Array.Empty<string>();
public static CertCcCursor Empty { get; } = new( public static CertCcCursor Empty { get; } = new(
TimeWindowCursorState.Empty, TimeWindowCursorState.Empty,
EmptyGuidArray, EmptyGuidArray,
EmptyStringArray, EmptyStringArray,
EmptyGuidArray, EmptyGuidArray,
EmptyGuidArray, EmptyGuidArray,
null); null);
public BsonDocument ToBsonDocument() public BsonDocument ToBsonDocument()
{ {
var document = new BsonDocument(); var document = new BsonDocument();
var summary = new BsonDocument(); var summary = new BsonDocument();
SummaryState.WriteTo(summary, "start", "end"); SummaryState.WriteTo(summary, "start", "end");
document["summary"] = summary; document["summary"] = summary;
document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString())); document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString()));
document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note)); document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note));
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())); document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString()));
document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())); document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString()));
if (LastRun.HasValue) if (LastRun.HasValue)
{ {
document["lastRun"] = LastRun.Value.UtcDateTime; document["lastRun"] = LastRun.Value.UtcDateTime;
} }
return document; return document;
} }
public static CertCcCursor FromBson(BsonDocument? document) public static CertCcCursor FromBson(BsonDocument? document)
{ {
if (document is null || document.ElementCount == 0) if (document is null || document.ElementCount == 0)
{ {
return Empty; return Empty;
} }
TimeWindowCursorState summaryState = TimeWindowCursorState.Empty; TimeWindowCursorState summaryState = TimeWindowCursorState.Empty;
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument) if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument)
{ {
summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end"); summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end");
} }
var pendingSummaries = ReadGuidArray(document, "pendingSummaries"); var pendingSummaries = ReadGuidArray(document, "pendingSummaries");
var pendingNotes = ReadStringArray(document, "pendingNotes"); var pendingNotes = ReadStringArray(document, "pendingNotes");
var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings"); var pendingMappings = ReadGuidArray(document, "pendingMappings");
DateTimeOffset? lastRun = null; DateTimeOffset? lastRun = null;
if (document.TryGetValue("lastRun", out var lastRunValue)) if (document.TryGetValue("lastRun", out var lastRunValue))
{ {
lastRun = lastRunValue.BsonType switch lastRun = lastRunValue.BsonType switch
{ {
BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc), BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(), BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null, _ => null,
}; };
} }
return new CertCcCursor(summaryState, pendingSummaries, pendingNotes, pendingDocuments, pendingMappings, lastRun); return new CertCcCursor(summaryState, pendingSummaries, pendingNotes, pendingDocuments, pendingMappings, lastRun);
} }
public CertCcCursor WithSummaryState(TimeWindowCursorState state) public CertCcCursor WithSummaryState(TimeWindowCursorState state)
=> this with { SummaryState = state ?? TimeWindowCursorState.Empty }; => this with { SummaryState = state ?? TimeWindowCursorState.Empty };
public CertCcCursor WithPendingSummaries(IEnumerable<Guid>? ids) public CertCcCursor WithPendingSummaries(IEnumerable<Guid>? ids)
=> this with { PendingSummaries = NormalizeGuidSet(ids) }; => this with { PendingSummaries = NormalizeGuidSet(ids) };
public CertCcCursor WithPendingNotes(IEnumerable<string>? notes) public CertCcCursor WithPendingNotes(IEnumerable<string>? notes)
=> this with { PendingNotes = NormalizeStringSet(notes) }; => this with { PendingNotes = NormalizeStringSet(notes) };
public CertCcCursor WithPendingDocuments(IEnumerable<Guid>? ids) public CertCcCursor WithPendingDocuments(IEnumerable<Guid>? ids)
=> this with { PendingDocuments = NormalizeGuidSet(ids) }; => this with { PendingDocuments = NormalizeGuidSet(ids) };
public CertCcCursor WithPendingMappings(IEnumerable<Guid>? ids) public CertCcCursor WithPendingMappings(IEnumerable<Guid>? ids)
=> this with { PendingMappings = NormalizeGuidSet(ids) }; => this with { PendingMappings = NormalizeGuidSet(ids) };
public CertCcCursor WithLastRun(DateTimeOffset? timestamp) public CertCcCursor WithLastRun(DateTimeOffset? timestamp)
=> this with { LastRun = timestamp }; => this with { LastRun = timestamp };
private static Guid[] ReadGuidArray(BsonDocument document, string field) private static Guid[] ReadGuidArray(BsonDocument document, string field)
{ {
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
{ {
return EmptyGuidArray; return EmptyGuidArray;
} }
var results = new List<Guid>(array.Count); var results = new List<Guid>(array.Count);
foreach (var element in array) foreach (var element in array)
{ {
if (element is BsonString bsonString && Guid.TryParse(bsonString.AsString, out var parsed)) if (TryReadGuid(element, out var parsed))
{ {
results.Add(parsed); results.Add(parsed);
continue; }
} }
if (element is BsonBinaryData binary && binary.GuidRepresentation != GuidRepresentation.Unspecified) return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray();
{ }
results.Add(binary.ToGuid());
} private static string[] ReadStringArray(BsonDocument document, string field)
} {
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray(); {
} return EmptyStringArray;
}
private static string[] ReadStringArray(BsonDocument document, string field)
{ var results = new List<string>(array.Count);
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) foreach (var element in array)
{ {
return EmptyStringArray; switch (element)
} {
case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
var results = new List<string>(array.Count); results.Add(bsonString.AsString.Trim());
foreach (var element in array) break;
{ case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
switch (element) results.Add(inner.AsString.Trim());
{ break;
case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString): }
results.Add(bsonString.AsString.Trim()); }
break;
case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString: return results.Count == 0
results.Add(inner.AsString.Trim()); ? EmptyStringArray
break; : results
} .Where(static value => !string.IsNullOrWhiteSpace(value))
} .Select(static value => value.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
return results.Count == 0 .ToArray();
? EmptyStringArray }
: results
.Where(static value => !string.IsNullOrWhiteSpace(value)) private static bool TryReadGuid(BsonValue value, out Guid guid)
.Select(static value => value.Trim()) {
.Distinct(StringComparer.OrdinalIgnoreCase) if (value is BsonString bsonString && Guid.TryParse(bsonString.AsString, out guid))
.ToArray(); {
} return true;
}
private static Guid[] NormalizeGuidSet(IEnumerable<Guid>? ids)
=> ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray; if (value is BsonBinaryData binary)
{
private static string[] NormalizeStringSet(IEnumerable<string>? values) try
=> values is null {
? EmptyStringArray guid = binary.ToGuid();
: values return true;
.Where(static value => !string.IsNullOrWhiteSpace(value)) }
.Select(static value => value.Trim()) catch (FormatException)
.Distinct(StringComparer.OrdinalIgnoreCase) {
.ToArray(); // ignore and fall back to byte array parsing
} }
var bytes = binary.AsByteArray;
if (bytes.Length == 16)
{
guid = new Guid(bytes);
return true;
}
}
guid = default;
return false;
}
private static Guid[] NormalizeGuidSet(IEnumerable<Guid>? ids)
=> ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray;
private static string[] NormalizeStringSet(IEnumerable<string>? values)
=> values is null
? EmptyStringArray
: values
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
}

View File

@@ -0,0 +1,124 @@
using System.Globalization;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Configuration;
public sealed class CiscoOptions
{
public const string HttpClientName = "concelier.source.vndr.cisco";
public const string AuthHttpClientName = "concelier.source.vndr.cisco.auth";
public Uri BaseUri { get; set; } = new("https://api.cisco.com/security/advisories/v2/", UriKind.Absolute);
public Uri TokenEndpoint { get; set; } = new("https://id.cisco.com/oauth2/default/v1/token", UriKind.Absolute);
public string ClientId { get; set; } = string.Empty;
public string ClientSecret { get; set; } = string.Empty;
public int PageSize { get; set; } = 100;
public int MaxPagesPerFetch { get; set; } = 5;
public int MaxAdvisoriesPerFetch { get; set; } = 200;
public TimeSpan InitialBackfillWindow { get; set; } = TimeSpan.FromDays(30);
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
public TimeSpan TokenRefreshSkew { get; set; } = TimeSpan.FromMinutes(1);
public string LastModifiedPathTemplate { get; set; } = "advisories/lastmodified/{0}";
public void Validate()
{
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("Cisco BaseUri must be an absolute URI.");
}
if (TokenEndpoint is null || !TokenEndpoint.IsAbsoluteUri)
{
throw new InvalidOperationException("Cisco TokenEndpoint must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("Cisco clientId must be configured.");
}
if (string.IsNullOrWhiteSpace(ClientSecret))
{
throw new InvalidOperationException("Cisco clientSecret must be configured.");
}
if (PageSize is < 1 or > 100)
{
throw new InvalidOperationException("Cisco PageSize must be between 1 and 100.");
}
if (MaxPagesPerFetch <= 0)
{
throw new InvalidOperationException("Cisco MaxPagesPerFetch must be greater than zero.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException("Cisco MaxAdvisoriesPerFetch must be greater than zero.");
}
if (InitialBackfillWindow <= TimeSpan.Zero)
{
throw new InvalidOperationException("Cisco InitialBackfillWindow must be positive.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("Cisco RequestDelay cannot be negative.");
}
if (RequestTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("Cisco RequestTimeout must be positive.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException("Cisco FailureBackoff must be positive.");
}
if (TokenRefreshSkew < TimeSpan.FromSeconds(5))
{
throw new InvalidOperationException("Cisco TokenRefreshSkew must be at least 5 seconds.");
}
if (string.IsNullOrWhiteSpace(LastModifiedPathTemplate))
{
throw new InvalidOperationException("Cisco LastModifiedPathTemplate must be configured.");
}
}
public Uri BuildLastModifiedUri(DateOnly date, int pageIndex, int pageSize)
{
if (pageIndex < 1)
{
throw new ArgumentOutOfRangeException(nameof(pageIndex), pageIndex, "Page index must be >= 1.");
}
if (pageSize is < 1 or > 100)
{
throw new ArgumentOutOfRangeException(nameof(pageSize), pageSize, "Page size must be between 1 and 100.");
}
var path = string.Format(CultureInfo.InvariantCulture, LastModifiedPathTemplate, date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
var builder = new UriBuilder(BaseUri);
var basePath = builder.Path.TrimEnd('/');
builder.Path = $"{basePath}/{path}".Replace("//", "/", StringComparison.Ordinal);
var query = $"pageIndex={pageIndex.ToString(CultureInfo.InvariantCulture)}&pageSize={pageSize.ToString(CultureInfo.InvariantCulture)}";
builder.Query = string.IsNullOrEmpty(builder.Query) ? query : builder.Query.TrimStart('?') + "&" + query;
return builder.Uri;
}
}

View File

@@ -0,0 +1,145 @@
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal sealed class CiscoAccessTokenProvider : IDisposable
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly IOptionsMonitor<CiscoOptions> _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<CiscoAccessTokenProvider> _logger;
private readonly SemaphoreSlim _refreshLock = new(1, 1);
private volatile AccessToken? _cached;
private bool _disposed;
public CiscoAccessTokenProvider(
IHttpClientFactory httpClientFactory,
IOptionsMonitor<CiscoOptions> options,
TimeProvider? timeProvider,
ILogger<CiscoAccessTokenProvider> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = options ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<string> GetTokenAsync(CancellationToken cancellationToken)
=> await GetTokenInternalAsync(forceRefresh: false, cancellationToken).ConfigureAwait(false);
public void Invalidate()
=> _cached = null;
private async Task<string> GetTokenInternalAsync(bool forceRefresh, CancellationToken cancellationToken)
{
ThrowIfDisposed();
var options = _options.CurrentValue;
var now = _timeProvider.GetUtcNow();
var cached = _cached;
if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew)
{
return cached.Value;
}
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
cached = _cached;
now = _timeProvider.GetUtcNow();
if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew)
{
return cached.Value;
}
var fresh = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false);
_cached = fresh;
return fresh.Value;
}
finally
{
_refreshLock.Release();
}
}
private async Task<AccessToken> RequestTokenAsync(CiscoOptions options, CancellationToken cancellationToken)
{
var client = _httpClientFactory.CreateClient(CiscoOptions.AuthHttpClientName);
client.Timeout = options.RequestTimeout;
using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint);
request.Headers.Accept.Clear();
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var content = new FormUrlEncodedContent(new Dictionary<string, string>
{
["grant_type"] = "client_credentials",
["client_id"] = options.ClientId,
["client_secret"] = options.ClientSecret,
});
request.Content = content;
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var message = $"Cisco OAuth token request failed with status {(int)response.StatusCode} {response.StatusCode}.";
_logger.LogError("Cisco openVuln token request failed: {Message}; response={Preview}", message, preview);
throw new HttpRequestException(message);
}
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
var payload = await JsonSerializer.DeserializeAsync<TokenResponse>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false);
if (payload is null || string.IsNullOrWhiteSpace(payload.AccessToken))
{
throw new InvalidOperationException("Cisco OAuth token response did not include an access token.");
}
var expiresIn = payload.ExpiresIn > 0 ? TimeSpan.FromSeconds(payload.ExpiresIn) : TimeSpan.FromHours(1);
var now = _timeProvider.GetUtcNow();
var expiresAt = now + expiresIn;
_logger.LogInformation("Cisco openVuln token issued; expires in {ExpiresIn}", expiresIn);
return new AccessToken(payload.AccessToken, expiresAt);
}
public async Task<string> RefreshAsync(CancellationToken cancellationToken)
=> await GetTokenInternalAsync(forceRefresh: true, cancellationToken).ConfigureAwait(false);
private void ThrowIfDisposed()
{
if (_disposed)
{
throw new ObjectDisposedException(nameof(CiscoAccessTokenProvider));
}
}
public void Dispose()
{
if (_disposed)
{
return;
}
_refreshLock.Dispose();
_disposed = true;
}
private sealed record AccessToken(string Value, DateTimeOffset ExpiresAt);
private sealed record TokenResponse(
[property: JsonPropertyName("access_token")] string AccessToken,
[property: JsonPropertyName("expires_in")] int ExpiresIn,
[property: JsonPropertyName("token_type")] string? TokenType);
}

View File

@@ -0,0 +1,25 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public sealed record CiscoAdvisoryDto(
string AdvisoryId,
string Title,
string? Summary,
string? Severity,
DateTimeOffset? Published,
DateTimeOffset? Updated,
string? PublicationUrl,
string? CsafUrl,
string? CvrfUrl,
double? CvssBaseScore,
IReadOnlyList<string> Cves,
IReadOnlyList<string> BugIds,
IReadOnlyList<CiscoAffectedProductDto> Products);
public sealed record CiscoAffectedProductDto(
string Name,
string? ProductId,
string? Version,
IReadOnlyCollection<string> Statuses);

View File

@@ -0,0 +1,64 @@
using System;
using System.IO;
using System.Net.Http;
using System.Text;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public interface ICiscoCsafClient
{
Task<string?> TryFetchAsync(string? url, CancellationToken cancellationToken);
}
public class CiscoCsafClient : ICiscoCsafClient
{
private static readonly string[] AcceptHeaders = { "application/json", "application/csaf+json", "application/vnd.cisco.csaf+json" };
private readonly SourceFetchService _fetchService;
private readonly ILogger<CiscoCsafClient> _logger;
public CiscoCsafClient(SourceFetchService fetchService, ILogger<CiscoCsafClient> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public virtual async Task<string?> TryFetchAsync(string? url, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(url))
{
return null;
}
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
{
_logger.LogWarning("Cisco CSAF URL '{Url}' is not a valid absolute URI.", url);
return null;
}
try
{
var request = new SourceFetchRequest(CiscoOptions.HttpClientName, VndrCiscoConnectorPlugin.SourceName, uri)
{
AcceptHeaders = AcceptHeaders,
};
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogWarning("Cisco CSAF download returned status {Status} for {Url}", result.StatusCode, url);
return null;
}
return System.Text.Encoding.UTF8.GetString(result.Content);
}
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or IOException or InvalidOperationException)
{
_logger.LogWarning(ex, "Cisco CSAF download failed for {Url}", url);
return null;
}
}
}

View File

@@ -0,0 +1,9 @@
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal sealed record CiscoCsafData(
IReadOnlyDictionary<string, CiscoCsafProduct> Products,
IReadOnlyDictionary<string, IReadOnlyCollection<string>> ProductStatuses);
internal sealed record CiscoCsafProduct(string ProductId, string Name);

View File

@@ -0,0 +1,123 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal static class CiscoCsafParser
{
public static CiscoCsafData Parse(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return new CiscoCsafData(
Products: new Dictionary<string, CiscoCsafProduct>(0, StringComparer.OrdinalIgnoreCase),
ProductStatuses: new Dictionary<string, IReadOnlyCollection<string>>(0, StringComparer.OrdinalIgnoreCase));
}
using var document = JsonDocument.Parse(content);
var root = document.RootElement;
var products = ParseProducts(root);
var statuses = ParseStatuses(root);
return new CiscoCsafData(products, statuses);
}
private static IReadOnlyDictionary<string, CiscoCsafProduct> ParseProducts(JsonElement root)
{
var dictionary = new Dictionary<string, CiscoCsafProduct>(StringComparer.OrdinalIgnoreCase);
if (!root.TryGetProperty("product_tree", out var productTree))
{
return dictionary;
}
if (productTree.TryGetProperty("full_product_names", out var fullProductNames)
&& fullProductNames.ValueKind == JsonValueKind.Array)
{
foreach (var entry in fullProductNames.EnumerateArray())
{
var productId = entry.TryGetProperty("product_id", out var idElement) && idElement.ValueKind == JsonValueKind.String
? idElement.GetString()
: null;
if (string.IsNullOrWhiteSpace(productId))
{
continue;
}
var name = entry.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String
? nameElement.GetString()
: null;
if (string.IsNullOrWhiteSpace(name))
{
name = productId;
}
dictionary[productId] = new CiscoCsafProduct(productId, name);
}
}
return dictionary;
}
private static IReadOnlyDictionary<string, IReadOnlyCollection<string>> ParseStatuses(JsonElement root)
{
var map = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase);
if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities)
|| vulnerabilities.ValueKind != JsonValueKind.Array)
{
return map.ToDictionary(
static kvp => kvp.Key,
static kvp => (IReadOnlyCollection<string>)kvp.Value.ToArray(),
StringComparer.OrdinalIgnoreCase);
}
foreach (var vulnerability in vulnerabilities.EnumerateArray())
{
if (!vulnerability.TryGetProperty("product_status", out var productStatus)
|| productStatus.ValueKind != JsonValueKind.Object)
{
continue;
}
foreach (var property in productStatus.EnumerateObject())
{
var statusLabel = property.Name;
if (property.Value.ValueKind != JsonValueKind.Array)
{
continue;
}
foreach (var productIdElement in property.Value.EnumerateArray())
{
if (productIdElement.ValueKind != JsonValueKind.String)
{
continue;
}
var productId = productIdElement.GetString();
if (string.IsNullOrWhiteSpace(productId))
{
continue;
}
if (!map.TryGetValue(productId, out var set))
{
set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
map[productId] = set;
}
set.Add(statusLabel);
}
}
}
return map.ToDictionary(
static kvp => kvp.Key,
static kvp => (IReadOnlyCollection<string>)kvp.Value.ToArray(),
StringComparer.OrdinalIgnoreCase);
}
}

View File

@@ -0,0 +1,101 @@
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal sealed record CiscoCursor(
DateTimeOffset? LastModified,
string? LastAdvisoryId,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
public static CiscoCursor Empty { get; } = new(null, null, EmptyGuidCollection, EmptyGuidCollection);
public BsonDocument ToBson()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastModified.HasValue)
{
document["lastModified"] = LastModified.Value.UtcDateTime;
}
if (!string.IsNullOrWhiteSpace(LastAdvisoryId))
{
document["lastAdvisoryId"] = LastAdvisoryId;
}
return document;
}
public static CiscoCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
DateTimeOffset? lastModified = null;
if (document.TryGetValue("lastModified", out var lastModifiedValue))
{
lastModified = lastModifiedValue.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
string? lastAdvisoryId = null;
if (document.TryGetValue("lastAdvisoryId", out var idValue) && idValue.BsonType == BsonType.String)
{
var value = idValue.AsString.Trim();
if (value.Length > 0)
{
lastAdvisoryId = value;
}
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
return new CiscoCursor(lastModified, lastAdvisoryId, pendingDocuments, pendingMappings);
}
public CiscoCursor WithCheckpoint(DateTimeOffset lastModified, string advisoryId)
=> this with
{
LastModified = lastModified.ToUniversalTime(),
LastAdvisoryId = string.IsNullOrWhiteSpace(advisoryId) ? null : advisoryId.Trim(),
};
public CiscoCursor WithPendingDocuments(IEnumerable<Guid>? documents)
=> this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidCollection };
public CiscoCursor WithPendingMappings(IEnumerable<Guid>? mappings)
=> this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidCollection };
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string key)
{
if (!document.TryGetValue(key, out var value) || value is not BsonArray array)
{
return EmptyGuidCollection;
}
var results = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element.ToString(), out var guid))
{
results.Add(guid);
}
}
return results;
}
}

View File

@@ -0,0 +1,82 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public sealed class CiscoDiagnostics : IDisposable
{
public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Cisco";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _fetchDocuments;
private readonly Counter<long> _fetchFailures;
private readonly Counter<long> _fetchUnchanged;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAffected;
public CiscoDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_fetchDocuments = _meter.CreateCounter<long>(
name: "cisco.fetch.documents",
unit: "documents",
description: "Number of Cisco advisories fetched.");
_fetchFailures = _meter.CreateCounter<long>(
name: "cisco.fetch.failures",
unit: "operations",
description: "Number of Cisco fetch failures.");
_fetchUnchanged = _meter.CreateCounter<long>(
name: "cisco.fetch.unchanged",
unit: "documents",
description: "Number of Cisco advisories skipped because they were unchanged.");
_parseSuccess = _meter.CreateCounter<long>(
name: "cisco.parse.success",
unit: "documents",
description: "Number of Cisco documents parsed successfully.");
_parseFailures = _meter.CreateCounter<long>(
name: "cisco.parse.failures",
unit: "documents",
description: "Number of Cisco documents that failed to parse.");
_mapSuccess = _meter.CreateCounter<long>(
name: "cisco.map.success",
unit: "documents",
description: "Number of Cisco advisories mapped successfully.");
_mapFailures = _meter.CreateCounter<long>(
name: "cisco.map.failures",
unit: "documents",
description: "Number of Cisco advisories that failed to map to canonical form.");
_mapAffected = _meter.CreateHistogram<long>(
name: "cisco.map.affected.packages",
unit: "packages",
description: "Distribution of affected package counts emitted per Cisco advisory.");
}
public Meter Meter => _meter;
public void FetchDocument() => _fetchDocuments.Add(1);
public void FetchFailure() => _fetchFailures.Add(1);
public void FetchUnchanged() => _fetchUnchanged.Add(1);
public void ParseSuccess() => _parseSuccess.Add(1);
public void ParseFailure() => _parseFailures.Add(1);
public void MapSuccess() => _mapSuccess.Add(1);
public void MapFailure() => _mapFailures.Add(1);
public void MapAffected(int count)
{
if (count >= 0)
{
_mapAffected.Record(count);
}
}
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,190 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public class CiscoDtoFactory
{
private readonly ICiscoCsafClient _csafClient;
private readonly ILogger<CiscoDtoFactory> _logger;
public CiscoDtoFactory(ICiscoCsafClient csafClient, ILogger<CiscoDtoFactory> logger)
{
_csafClient = csafClient ?? throw new ArgumentNullException(nameof(csafClient));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<CiscoAdvisoryDto> CreateAsync(CiscoRawAdvisory raw, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(raw);
var advisoryId = raw.AdvisoryId?.Trim();
if (string.IsNullOrWhiteSpace(advisoryId))
{
throw new InvalidOperationException("Cisco advisory is missing advisoryId.");
}
var title = string.IsNullOrWhiteSpace(raw.AdvisoryTitle) ? advisoryId : raw.AdvisoryTitle!.Trim();
var severity = SeverityNormalization.Normalize(raw.Sir);
var published = ParseDate(raw.FirstPublished);
var updated = ParseDate(raw.LastUpdated);
CiscoCsafData? csafData = null;
if (!string.IsNullOrWhiteSpace(raw.CsafUrl))
{
var csafContent = await _csafClient.TryFetchAsync(raw.CsafUrl, cancellationToken).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(csafContent))
{
try
{
csafData = CiscoCsafParser.Parse(csafContent!);
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Cisco CSAF payload parsing failed for {AdvisoryId}", advisoryId);
}
}
}
var products = BuildProducts(raw, csafData);
var cves = NormalizeList(raw.Cves);
var bugIds = NormalizeList(raw.BugIds);
var cvss = ParseDouble(raw.CvssBaseScore);
return new CiscoAdvisoryDto(
AdvisoryId: advisoryId,
Title: title,
Summary: string.IsNullOrWhiteSpace(raw.Summary) ? null : raw.Summary!.Trim(),
Severity: severity,
Published: published,
Updated: updated,
PublicationUrl: NormalizeUrl(raw.PublicationUrl),
CsafUrl: NormalizeUrl(raw.CsafUrl),
CvrfUrl: NormalizeUrl(raw.CvrfUrl),
CvssBaseScore: cvss,
Cves: cves,
BugIds: bugIds,
Products: products);
}
private static IReadOnlyList<CiscoAffectedProductDto> BuildProducts(CiscoRawAdvisory raw, CiscoCsafData? csafData)
{
var map = new Dictionary<string, CiscoAffectedProductDto>(StringComparer.OrdinalIgnoreCase);
if (csafData is not null)
{
foreach (var entry in csafData.ProductStatuses)
{
var productId = entry.Key;
var name = csafData.Products.TryGetValue(productId, out var product)
? product.Name
: productId;
var statuses = NormalizeStatuses(entry.Value);
map[name] = new CiscoAffectedProductDto(
Name: name,
ProductId: productId,
Version: raw.Version?.Trim(),
Statuses: statuses);
}
}
var rawProducts = NormalizeList(raw.ProductNames);
foreach (var productName in rawProducts)
{
if (map.ContainsKey(productName))
{
continue;
}
map[productName] = new CiscoAffectedProductDto(
Name: productName,
ProductId: null,
Version: raw.Version?.Trim(),
Statuses: new[] { AffectedPackageStatusCatalog.KnownAffected });
}
return map.Count == 0
? Array.Empty<CiscoAffectedProductDto>()
: map.Values
.OrderBy(static p => p.Name, StringComparer.OrdinalIgnoreCase)
.ThenBy(static p => p.ProductId, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyCollection<string> NormalizeStatuses(IEnumerable<string> statuses)
{
var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var status in statuses)
{
if (AffectedPackageStatusCatalog.TryNormalize(status, out var normalized))
{
set.Add(normalized);
}
else if (!string.IsNullOrWhiteSpace(status))
{
set.Add(status.Trim().ToLowerInvariant());
}
}
if (set.Count == 0)
{
set.Add(AffectedPackageStatusCatalog.KnownAffected);
}
return set;
}
private static IReadOnlyList<string> NormalizeList(IEnumerable<string>? items)
{
if (items is null)
{
return Array.Empty<string>();
}
var set = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var item in items)
{
if (!string.IsNullOrWhiteSpace(item))
{
set.Add(item.Trim());
}
}
return set.Count == 0 ? Array.Empty<string>() : set.ToArray();
}
private static double? ParseDouble(string? value)
=> double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)
? parsed
: null;
private static DateTimeOffset? ParseDate(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return parsed.ToUniversalTime();
}
return null;
}
private static string? NormalizeUrl(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return Uri.TryCreate(value.Trim(), UriKind.Absolute, out var uri) ? uri.ToString() : null;
}
}

View File

@@ -0,0 +1,263 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common.Packages;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public static class CiscoMapper
{
public static Advisory Map(CiscoAdvisoryDto dto, DocumentRecord document, DtoRecord dtoRecord)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
ArgumentNullException.ThrowIfNull(dtoRecord);
var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime();
var fetchProvenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"document",
document.Uri,
document.FetchedAt.ToUniversalTime());
var mapProvenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"map",
dto.AdvisoryId,
recordedAt);
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var affected = BuildAffectedPackages(dto, recordedAt);
return new Advisory(
advisoryKey: dto.AdvisoryId,
title: dto.Title,
summary: dto.Summary,
language: "en",
published: dto.Published,
modified: dto.Updated,
severity: dto.Severity,
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affected,
cvssMetrics: Array.Empty<CvssMetric>(),
provenance: new[] { fetchProvenance, mapProvenance });
}
private static IReadOnlyList<string> BuildAliases(CiscoAdvisoryDto dto)
{
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
dto.AdvisoryId,
};
foreach (var cve in dto.Cves)
{
if (!string.IsNullOrWhiteSpace(cve))
{
set.Add(cve.Trim());
}
}
foreach (var bugId in dto.BugIds)
{
if (!string.IsNullOrWhiteSpace(bugId))
{
set.Add(bugId.Trim());
}
}
if (dto.PublicationUrl is not null)
{
set.Add(dto.PublicationUrl);
}
return set.Count == 0
? Array.Empty<string>()
: set.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(CiscoAdvisoryDto dto, DateTimeOffset recordedAt)
{
var list = new List<AdvisoryReference>(3);
AddReference(list, dto.PublicationUrl, "publication", recordedAt);
AddReference(list, dto.CvrfUrl, "cvrf", recordedAt);
AddReference(list, dto.CsafUrl, "csaf", recordedAt);
return list.Count == 0
? Array.Empty<AdvisoryReference>()
: list.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static void AddReference(ICollection<AdvisoryReference> references, string? url, string kind, DateTimeOffset recordedAt)
{
if (string.IsNullOrWhiteSpace(url))
{
return;
}
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
{
return;
}
var provenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
$"reference:{kind}",
uri.ToString(),
recordedAt);
try
{
references.Add(new AdvisoryReference(
url: uri.ToString(),
kind: kind,
sourceTag: null,
summary: null,
provenance: provenance));
}
catch (ArgumentException)
{
// ignore invalid URLs
}
}
private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(CiscoAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
if (string.IsNullOrWhiteSpace(product.Name))
{
continue;
}
var range = BuildVersionRange(product, recordedAt);
var statuses = BuildStatuses(product, recordedAt);
var provenance = new[]
{
new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"affected",
product.ProductId ?? product.Name,
recordedAt),
};
packages.Add(new AffectedPackage(
type: AffectedPackageTypes.Vendor,
identifier: product.Name,
platform: null,
versionRanges: range is null ? Array.Empty<AffectedVersionRange>() : new[] { range },
statuses: statuses,
provenance: provenance,
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
}
return packages.Count == 0
? Array.Empty<AffectedPackage>()
: packages.OrderBy(static p => p.Identifier, StringComparer.OrdinalIgnoreCase).ToArray();
}
private static AffectedVersionRange? BuildVersionRange(CiscoAffectedProductDto product, DateTimeOffset recordedAt)
{
if (string.IsNullOrWhiteSpace(product.Version))
{
return null;
}
var version = product.Version.Trim();
RangePrimitives? primitives = null;
string rangeKind = "vendor";
string? rangeExpression = version;
if (PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized))
{
var semver = new SemVerPrimitive(
Introduced: null,
IntroducedInclusive: true,
Fixed: null,
FixedInclusive: false,
LastAffected: null,
LastAffectedInclusive: true,
ConstraintExpression: null,
ExactValue: normalized);
primitives = new RangePrimitives(semver, null, null, BuildVendorExtensions(product));
rangeKind = "semver";
rangeExpression = normalized;
}
else
{
primitives = new RangePrimitives(null, null, null, BuildVendorExtensions(product, includeVersion: true));
}
var provenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"range",
product.ProductId ?? product.Name,
recordedAt);
return new AffectedVersionRange(
rangeKind: rangeKind,
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: rangeExpression,
provenance: provenance,
primitives: primitives);
}
private static IReadOnlyDictionary<string, string>? BuildVendorExtensions(CiscoAffectedProductDto product, bool includeVersion = false)
{
var dictionary = new Dictionary<string, string>(StringComparer.Ordinal);
if (!string.IsNullOrWhiteSpace(product.ProductId))
{
dictionary["cisco.productId"] = product.ProductId!;
}
if (includeVersion && !string.IsNullOrWhiteSpace(product.Version))
{
dictionary["cisco.version.raw"] = product.Version!;
}
return dictionary.Count == 0 ? null : dictionary;
}
private static IReadOnlyList<AffectedPackageStatus> BuildStatuses(CiscoAffectedProductDto product, DateTimeOffset recordedAt)
{
if (product.Statuses is null || product.Statuses.Count == 0)
{
return Array.Empty<AffectedPackageStatus>();
}
var list = new List<AffectedPackageStatus>(product.Statuses.Count);
foreach (var status in product.Statuses)
{
if (!AffectedPackageStatusCatalog.TryNormalize(status, out var normalized)
|| string.IsNullOrWhiteSpace(normalized))
{
continue;
}
var provenance = new AdvisoryProvenance(
VndrCiscoConnectorPlugin.SourceName,
"status",
product.ProductId ?? product.Name,
recordedAt);
list.Add(new AffectedPackageStatus(normalized, provenance));
}
return list.Count == 0 ? Array.Empty<AffectedPackageStatus>() : list;
}
}

View File

@@ -0,0 +1,101 @@
using System.IO;
using System.Net;
using System.Net.Http.Headers;
using Microsoft.Extensions.Logging;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
internal sealed class CiscoOAuthMessageHandler : DelegatingHandler
{
private readonly CiscoAccessTokenProvider _tokenProvider;
private readonly ILogger<CiscoOAuthMessageHandler> _logger;
public CiscoOAuthMessageHandler(
CiscoAccessTokenProvider tokenProvider,
ILogger<CiscoOAuthMessageHandler> logger)
{
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
HttpRequestMessage? retryTemplate = null;
try
{
retryTemplate = await CloneRequestAsync(request, cancellationToken).ConfigureAwait(false);
}
catch (IOException)
{
// Unable to buffer content; retry will fail if needed.
retryTemplate = null;
}
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false));
var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (response.StatusCode != HttpStatusCode.Unauthorized)
{
return response;
}
response.Dispose();
_logger.LogWarning("Cisco openVuln request returned 401 Unauthorized; refreshing access token.");
await _tokenProvider.RefreshAsync(cancellationToken).ConfigureAwait(false);
if (retryTemplate is null)
{
_tokenProvider.Invalidate();
throw new HttpRequestException("Cisco openVuln request returned 401 Unauthorized and could not be retried.");
}
retryTemplate.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false));
try
{
var retryResponse = await base.SendAsync(retryTemplate, cancellationToken).ConfigureAwait(false);
if (retryResponse.StatusCode == HttpStatusCode.Unauthorized)
{
_tokenProvider.Invalidate();
}
return retryResponse;
}
finally
{
retryTemplate.Dispose();
}
}
private static async Task<HttpRequestMessage?> CloneRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var clone = new HttpRequestMessage(request.Method, request.RequestUri)
{
Version = request.Version,
VersionPolicy = request.VersionPolicy,
};
foreach (var header in request.Headers)
{
clone.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
if (request.Content is not null)
{
using var memory = new MemoryStream();
await request.Content.CopyToAsync(memory, cancellationToken).ConfigureAwait(false);
memory.Position = 0;
var buffer = memory.ToArray();
var contentClone = new ByteArrayContent(buffer);
foreach (var header in request.Content.Headers)
{
contentClone.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
clone.Content = contentClone;
}
return clone;
}
}

View File

@@ -0,0 +1,196 @@
using System.Diagnostics.CodeAnalysis;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public sealed class CiscoOpenVulnClient
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString,
};
private readonly SourceFetchService _fetchService;
private readonly IOptionsMonitor<CiscoOptions> _options;
private readonly ILogger<CiscoOpenVulnClient> _logger;
private readonly string _sourceName;
public CiscoOpenVulnClient(
SourceFetchService fetchService,
IOptionsMonitor<CiscoOptions> options,
ILogger<CiscoOpenVulnClient> logger,
string sourceName)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_sourceName = sourceName ?? throw new ArgumentNullException(nameof(sourceName));
}
internal async Task<CiscoAdvisoryPage?> FetchAsync(DateOnly date, int pageIndex, CancellationToken cancellationToken)
{
var options = _options.CurrentValue;
var requestUri = options.BuildLastModifiedUri(date, pageIndex, options.PageSize);
var request = new SourceFetchRequest(CiscoOptions.HttpClientName, _sourceName, requestUri)
{
AcceptHeaders = new[] { "application/json" },
TimeoutOverride = options.RequestTimeout,
};
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
if (!result.IsSuccess || result.Content is null)
{
_logger.LogDebug("Cisco openVuln request returned empty payload for {Uri} (status {Status})", requestUri, result.StatusCode);
return null;
}
return CiscoAdvisoryPage.Parse(result.Content);
}
}
internal sealed record CiscoAdvisoryPage(
IReadOnlyList<CiscoAdvisoryItem> Advisories,
CiscoPagination Pagination)
{
public bool HasMore => Pagination.PageIndex < Pagination.TotalPages;
public static CiscoAdvisoryPage Parse(byte[] content)
{
using var document = JsonDocument.Parse(content);
var root = document.RootElement;
var advisories = new List<CiscoAdvisoryItem>();
if (root.TryGetProperty("advisories", out var advisoriesElement) && advisoriesElement.ValueKind == JsonValueKind.Array)
{
foreach (var advisory in advisoriesElement.EnumerateArray())
{
if (!TryCreateItem(advisory, out var item))
{
continue;
}
advisories.Add(item);
}
}
var pagination = CiscoPagination.FromJson(root.TryGetProperty("pagination", out var paginationElement) ? paginationElement : default);
return new CiscoAdvisoryPage(advisories, pagination);
}
private static bool TryCreateItem(JsonElement advisory, [NotNullWhen(true)] out CiscoAdvisoryItem? item)
{
var rawJson = advisory.GetRawText();
var advisoryId = GetString(advisory, "advisoryId");
if (string.IsNullOrWhiteSpace(advisoryId))
{
item = null;
return false;
}
var lastUpdated = ParseDate(GetString(advisory, "lastUpdated"));
var firstPublished = ParseDate(GetString(advisory, "firstPublished"));
var severity = GetString(advisory, "sir");
var publicationUrl = GetString(advisory, "publicationUrl");
var csafUrl = GetString(advisory, "csafUrl");
var cvrfUrl = GetString(advisory, "cvrfUrl");
var cvss = GetString(advisory, "cvssBaseScore");
var cves = ReadStringArray(advisory, "cves");
var bugIds = ReadStringArray(advisory, "bugIDs");
var productNames = ReadStringArray(advisory, "productNames");
item = new CiscoAdvisoryItem(
advisoryId,
lastUpdated,
firstPublished,
severity,
publicationUrl,
csafUrl,
cvrfUrl,
cvss,
cves,
bugIds,
productNames,
rawJson);
return true;
}
private static string? GetString(JsonElement element, string propertyName)
=> element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.String
? value.GetString()
: null;
private static DateTimeOffset? ParseDate(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
if (DateTimeOffset.TryParse(value, out var parsed))
{
return parsed.ToUniversalTime();
}
return null;
}
private static IReadOnlyList<string> ReadStringArray(JsonElement element, string property)
{
if (!element.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.Array)
{
return Array.Empty<string>();
}
var results = new List<string>();
foreach (var child in value.EnumerateArray())
{
if (child.ValueKind == JsonValueKind.String)
{
var text = child.GetString();
if (!string.IsNullOrWhiteSpace(text))
{
results.Add(text.Trim());
}
}
}
return results;
}
}
internal sealed record CiscoAdvisoryItem(
string AdvisoryId,
DateTimeOffset? LastUpdated,
DateTimeOffset? FirstPublished,
string? Severity,
string? PublicationUrl,
string? CsafUrl,
string? CvrfUrl,
string? CvssBaseScore,
IReadOnlyList<string> Cves,
IReadOnlyList<string> BugIds,
IReadOnlyList<string> ProductNames,
string RawJson)
{
public byte[] GetRawBytes() => Encoding.UTF8.GetBytes(RawJson);
}
internal sealed record CiscoPagination(int PageIndex, int PageSize, int TotalPages, int TotalRecords)
{
public static CiscoPagination FromJson(JsonElement element)
{
var pageIndex = element.TryGetProperty("pageIndex", out var index) && index.TryGetInt32(out var parsedIndex) ? parsedIndex : 1;
var pageSize = element.TryGetProperty("pageSize", out var size) && size.TryGetInt32(out var parsedSize) ? parsedSize : 0;
var totalPages = element.TryGetProperty("totalPages", out var pages) && pages.TryGetInt32(out var parsedPages) ? parsedPages : pageIndex;
var totalRecords = element.TryGetProperty("totalRecords", out var records) && records.TryGetInt32(out var parsedRecords) ? parsedRecords : 0;
return new CiscoPagination(pageIndex, pageSize, totalPages, totalRecords);
}
}

View File

@@ -0,0 +1,64 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
public class CiscoRawAdvisory
{
[JsonPropertyName("advisoryId")]
public string? AdvisoryId { get; set; }
[JsonPropertyName("advisoryTitle")]
public string? AdvisoryTitle { get; set; }
[JsonPropertyName("publicationUrl")]
public string? PublicationUrl { get; set; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; set; }
[JsonPropertyName("csafUrl")]
public string? CsafUrl { get; set; }
[JsonPropertyName("summary")]
public string? Summary { get; set; }
[JsonPropertyName("sir")]
public string? Sir { get; set; }
[JsonPropertyName("firstPublished")]
public string? FirstPublished { get; set; }
[JsonPropertyName("lastUpdated")]
public string? LastUpdated { get; set; }
[JsonPropertyName("productNames")]
public List<string>? ProductNames { get; set; }
[JsonPropertyName("version")]
public string? Version { get; set; }
[JsonPropertyName("iosRelease")]
public string? IosRelease { get; set; }
[JsonPropertyName("cves")]
public List<string>? Cves { get; set; }
[JsonPropertyName("bugIDs")]
public List<string>? BugIds { get; set; }
[JsonPropertyName("cvssBaseScore")]
public string? CvssBaseScore { get; set; }
[JsonPropertyName("cvssTemporalScore")]
public string? CvssTemporalScore { get; set; }
[JsonPropertyName("cvssEnvironmentalScore")]
public string? CvssEnvironmentalScore { get; set; }
[JsonPropertyName("cvssBaseScoreVersion2")]
public string? CvssBaseScoreV2 { get; set; }
[JsonPropertyName("status")]
public string? Status { get; set; }
}

View File

@@ -0,0 +1,132 @@
using System.Globalization;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
public sealed class MsrcOptions
{
public const string HttpClientName = "concelier.source.vndr.msrc";
public const string TokenClientName = "concelier.source.vndr.msrc.token";
public Uri BaseUri { get; set; } = new("https://api.msrc.microsoft.com/sug/v2.0/", UriKind.Absolute);
public string Locale { get; set; } = "en-US";
public string ApiVersion { get; set; } = "2024-08-01";
/// <summary>
/// Azure AD tenant identifier used for client credential flow.
/// </summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>
/// Azure AD application (client) identifier.
/// </summary>
public string ClientId { get; set; } = string.Empty;
/// <summary>
/// Azure AD client secret used for token acquisition.
/// </summary>
public string ClientSecret { get; set; } = string.Empty;
/// <summary>
/// Scope requested during client-credential token acquisition.
/// </summary>
public string Scope { get; set; } = "api://api.msrc.microsoft.com/.default";
/// <summary>
/// Maximum advisories to fetch per cycle.
/// </summary>
public int MaxAdvisoriesPerFetch { get; set; } = 200;
/// <summary>
/// Page size used when iterating the MSRC API.
/// </summary>
public int PageSize { get; set; } = 100;
/// <summary>
/// Overlap window added when resuming from the last modified cursor.
/// </summary>
public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10);
/// <summary>
/// When enabled the connector downloads the CVRF artefact referenced by each advisory.
/// </summary>
public bool DownloadCvrf { get; set; } = false;
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Optional lower bound for the initial sync if the cursor is empty.
/// </summary>
public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30);
public void Validate()
{
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("MSRC base URI must be absolute.");
}
if (string.IsNullOrWhiteSpace(Locale))
{
throw new InvalidOperationException("Locale must be provided.");
}
if (!string.IsNullOrWhiteSpace(Locale) && !CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase)))
{
throw new InvalidOperationException($"Locale '{Locale}' is not recognised.");
}
if (string.IsNullOrWhiteSpace(ApiVersion))
{
throw new InvalidOperationException("API version must be provided.");
}
if (!Guid.TryParse(TenantId, out _))
{
throw new InvalidOperationException("TenantId must be a valid GUID.");
}
if (string.IsNullOrWhiteSpace(ClientId))
{
throw new InvalidOperationException("ClientId must be provided.");
}
if (string.IsNullOrWhiteSpace(ClientSecret))
{
throw new InvalidOperationException("ClientSecret must be provided.");
}
if (string.IsNullOrWhiteSpace(Scope))
{
throw new InvalidOperationException("Scope must be provided.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
}
if (PageSize <= 0 || PageSize > 500)
{
throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500.");
}
if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6))
{
throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
}
if (FailureBackoff <= TimeSpan.Zero)
{
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
}
}
}

View File

@@ -0,0 +1,49 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed record MsrcAdvisoryDto
{
public string AdvisoryId { get; init; } = string.Empty;
public string Title { get; init; } = string.Empty;
public string? Description { get; init; }
public string? Severity { get; init; }
public DateTimeOffset? ReleaseDate { get; init; }
public DateTimeOffset? LastModifiedDate { get; init; }
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
public IReadOnlyList<string> KbIds { get; init; } = Array.Empty<string>();
public IReadOnlyList<MsrcAdvisoryThreat> Threats { get; init; } = Array.Empty<MsrcAdvisoryThreat>();
public IReadOnlyList<MsrcAdvisoryRemediation> Remediations { get; init; } = Array.Empty<MsrcAdvisoryRemediation>();
public IReadOnlyList<MsrcAdvisoryProduct> Products { get; init; } = Array.Empty<MsrcAdvisoryProduct>();
public double? CvssBaseScore { get; init; }
public string? CvssVector { get; init; }
public string? ReleaseNoteUrl { get; init; }
public string? CvrfUrl { get; init; }
}
public sealed record MsrcAdvisoryThreat(string Type, string? Description, string? Severity);
public sealed record MsrcAdvisoryRemediation(string Type, string? Description, string? Url, string? Kb);
public sealed record MsrcAdvisoryProduct(
string Identifier,
string? ProductName,
string? Platform,
string? Architecture,
string? BuildNumber,
string? Cpe);

View File

@@ -0,0 +1,138 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed class MsrcApiClient
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true,
WriteIndented = false,
};
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMsrcTokenProvider _tokenProvider;
private readonly MsrcOptions _options;
private readonly ILogger<MsrcApiClient> _logger;
public MsrcApiClient(
IHttpClientFactory httpClientFactory,
IMsrcTokenProvider tokenProvider,
IOptions<MsrcOptions> options,
ILogger<MsrcApiClient> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<IReadOnlyList<MsrcVulnerabilitySummary>> FetchSummariesAsync(DateTimeOffset fromInclusive, DateTimeOffset toExclusive, CancellationToken cancellationToken)
{
var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false);
var results = new List<MsrcVulnerabilitySummary>();
var requestUri = BuildSummaryUri(fromInclusive, toExclusive);
while (requestUri is not null)
{
using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new HttpRequestException($"MSRC summary fetch failed with {(int)response.StatusCode}. Body: {preview}");
}
var payload = await response.Content.ReadFromJsonAsync<MsrcSummaryResponse>(SerializerOptions, cancellationToken).ConfigureAwait(false)
?? new MsrcSummaryResponse();
results.AddRange(payload.Value);
if (string.IsNullOrWhiteSpace(payload.NextLink))
{
break;
}
requestUri = new Uri(payload.NextLink, UriKind.Absolute);
}
return results;
}
public Uri BuildDetailUri(string vulnerabilityId)
{
var uri = CreateDetailUriInternal(vulnerabilityId);
return uri;
}
public async Task<byte[]> FetchDetailAsync(string vulnerabilityId, CancellationToken cancellationToken)
{
var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false);
var uri = CreateDetailUriInternal(vulnerabilityId);
using var request = new HttpRequestMessage(HttpMethod.Get, uri);
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new HttpRequestException($"MSRC detail fetch failed for {vulnerabilityId} with {(int)response.StatusCode}. Body: {preview}");
}
return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
}
private async Task<HttpClient> CreateAuthenticatedClientAsync(CancellationToken cancellationToken)
{
var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false);
var client = _httpClientFactory.CreateClient(MsrcOptions.HttpClientName);
client.DefaultRequestHeaders.Remove("Authorization");
client.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}");
client.DefaultRequestHeaders.Remove("Accept");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.DefaultRequestHeaders.Remove("api-version");
client.DefaultRequestHeaders.Add("api-version", _options.ApiVersion);
client.DefaultRequestHeaders.Remove("Accept-Language");
client.DefaultRequestHeaders.Add("Accept-Language", _options.Locale);
return client;
}
private Uri BuildSummaryUri(DateTimeOffset fromInclusive, DateTimeOffset toExclusive)
{
var builder = new StringBuilder();
builder.Append(_options.BaseUri.ToString().TrimEnd('/'));
builder.Append("/vulnerabilities?");
builder.Append("$top=").Append(_options.PageSize);
builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(fromInclusive.ToUniversalTime().ToString("O")));
builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(toExclusive.ToUniversalTime().ToString("O")));
builder.Append("&$orderby=lastModifiedDate");
builder.Append("&locale=").Append(Uri.EscapeDataString(_options.Locale));
builder.Append("&api-version=").Append(Uri.EscapeDataString(_options.ApiVersion));
return new Uri(builder.ToString(), UriKind.Absolute);
}
private Uri CreateDetailUriInternal(string vulnerabilityId)
{
if (string.IsNullOrWhiteSpace(vulnerabilityId))
{
throw new ArgumentException("Vulnerability identifier must be provided.", nameof(vulnerabilityId));
}
var baseUri = _options.BaseUri.ToString().TrimEnd('/');
var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(_options.ApiVersion)}&locale={Uri.EscapeDataString(_options.Locale)}";
return new Uri(path, UriKind.Absolute);
}
}

View File

@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using MongoDB.Bson;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
internal sealed record MsrcCursor(
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings,
DateTimeOffset? LastModifiedCursor)
{
private static readonly IReadOnlyCollection<Guid> EmptyGuidSet = Array.Empty<Guid>();
public static MsrcCursor Empty { get; } = new(EmptyGuidSet, EmptyGuidSet, null);
public MsrcCursor WithPendingDocuments(IEnumerable<Guid> documents)
=> this with { PendingDocuments = Distinct(documents) };
public MsrcCursor WithPendingMappings(IEnumerable<Guid> mappings)
=> this with { PendingMappings = Distinct(mappings) };
public MsrcCursor WithLastModifiedCursor(DateTimeOffset? timestamp)
=> this with { LastModifiedCursor = timestamp };
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastModifiedCursor.HasValue)
{
document["lastModifiedCursor"] = LastModifiedCursor.Value.UtcDateTime;
}
return document;
}
public static MsrcCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
var pendingMappings = ReadGuidArray(document, "pendingMappings");
var lastModified = document.TryGetValue("lastModifiedCursor", out var value)
? ParseDate(value)
: null;
return new MsrcCursor(pendingDocuments, pendingMappings, lastModified);
}
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
=> values?.Distinct().ToArray() ?? EmptyGuidSet;
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidSet;
}
var items = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element?.ToString(), out var id))
{
items.Add(id);
}
}
return items;
}
private static DateTimeOffset? ParseDate(BsonValue value)
=> value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}

View File

@@ -0,0 +1,113 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed record MsrcVulnerabilityDetailDto
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string VulnerabilityId { get; init; } = string.Empty;
[JsonPropertyName("cveNumber")]
public string? CveNumber { get; init; }
[JsonPropertyName("cveNumbers")]
public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>();
[JsonPropertyName("title")]
public string Title { get; init; } = string.Empty;
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("threats")]
public IReadOnlyList<MsrcThreatDto> Threats { get; init; } = Array.Empty<MsrcThreatDto>();
[JsonPropertyName("remediations")]
public IReadOnlyList<MsrcRemediationDto> Remediations { get; init; } = Array.Empty<MsrcRemediationDto>();
[JsonPropertyName("affectedProducts")]
public IReadOnlyList<MsrcAffectedProductDto> AffectedProducts { get; init; } = Array.Empty<MsrcAffectedProductDto>();
[JsonPropertyName("cvssV3")]
public MsrcCvssDto? Cvss { get; init; }
[JsonPropertyName("releaseNoteUrl")]
public string? ReleaseNoteUrl { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}
public sealed record MsrcThreatDto
{
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
}
public sealed record MsrcRemediationDto
{
[JsonPropertyName("id")]
public string? Id { get; init; }
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("url")]
public string? Url { get; init; }
[JsonPropertyName("kbNumber")]
public string? KbNumber { get; init; }
}
public sealed record MsrcAffectedProductDto
{
[JsonPropertyName("productId")]
public string? ProductId { get; init; }
[JsonPropertyName("productName")]
public string? ProductName { get; init; }
[JsonPropertyName("cpe")]
public string? Cpe { get; init; }
[JsonPropertyName("platform")]
public string? Platform { get; init; }
[JsonPropertyName("architecture")]
public string? Architecture { get; init; }
[JsonPropertyName("buildNumber")]
public string? BuildNumber { get; init; }
}
public sealed record MsrcCvssDto
{
[JsonPropertyName("baseScore")]
public double? BaseScore { get; init; }
[JsonPropertyName("vectorString")]
public string? VectorString { get; init; }
}

View File

@@ -0,0 +1,71 @@
using System;
using System.Linq;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed class MsrcDetailParser
{
public MsrcAdvisoryDto Parse(MsrcVulnerabilityDetailDto detail)
{
ArgumentNullException.ThrowIfNull(detail);
var advisoryId = string.IsNullOrWhiteSpace(detail.VulnerabilityId) ? detail.Id : detail.VulnerabilityId;
var cveIds = detail.CveNumbers?.Where(static c => !string.IsNullOrWhiteSpace(c)).Select(static c => c.Trim()).ToArray()
?? (string.IsNullOrWhiteSpace(detail.CveNumber) ? Array.Empty<string>() : new[] { detail.CveNumber! });
var kbIds = detail.Remediations?
.Where(static remediation => !string.IsNullOrWhiteSpace(remediation.KbNumber))
.Select(static remediation => remediation.KbNumber!.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? Array.Empty<string>();
return new MsrcAdvisoryDto
{
AdvisoryId = advisoryId,
Title = string.IsNullOrWhiteSpace(detail.Title) ? advisoryId : detail.Title.Trim(),
Description = detail.Description,
Severity = detail.Severity,
ReleaseDate = detail.ReleaseDate,
LastModifiedDate = detail.LastModifiedDate,
CveIds = cveIds,
KbIds = kbIds,
Threats = detail.Threats?.Select(static threat => new MsrcAdvisoryThreat(
threat.Type ?? "unspecified",
threat.Description,
threat.Severity)).ToArray() ?? Array.Empty<MsrcAdvisoryThreat>(),
Remediations = detail.Remediations?.Select(static remediation => new MsrcAdvisoryRemediation(
remediation.Type ?? "unspecified",
remediation.Description,
remediation.Url,
remediation.KbNumber)).ToArray() ?? Array.Empty<MsrcAdvisoryRemediation>(),
Products = detail.AffectedProducts?.Select(product =>
new MsrcAdvisoryProduct(
BuildProductIdentifier(product),
product.ProductName,
product.Platform,
product.Architecture,
product.BuildNumber,
product.Cpe)).ToArray() ?? Array.Empty<MsrcAdvisoryProduct>(),
CvssBaseScore = detail.Cvss?.BaseScore,
CvssVector = detail.Cvss?.VectorString,
ReleaseNoteUrl = detail.ReleaseNoteUrl,
CvrfUrl = detail.CvrfUrl,
};
}
private static string BuildProductIdentifier(MsrcAffectedProductDto product)
{
var name = string.IsNullOrWhiteSpace(product.ProductName) ? product.ProductId : product.ProductName;
if (string.IsNullOrWhiteSpace(name))
{
name = "Unknown Product";
}
if (!string.IsNullOrWhiteSpace(product.BuildNumber))
{
return $"{name} build {product.BuildNumber}";
}
return name;
}
}

View File

@@ -0,0 +1,129 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed class MsrcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.Vndr.Msrc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchFailures;
private readonly Histogram<long> _summaryItemCount;
private readonly Histogram<double> _summaryWindowHours;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchNotModified;
private readonly Counter<long> _detailFetchFailures;
private readonly Histogram<long> _detailEnqueued;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseProductCount;
private readonly Histogram<long> _parseKbCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAliasCount;
private readonly Histogram<long> _mapAffectedCount;
public MsrcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_summaryFetchAttempts = _meter.CreateCounter<long>("msrc.summary.fetch.attempts", "operations");
_summaryFetchSuccess = _meter.CreateCounter<long>("msrc.summary.fetch.success", "operations");
_summaryFetchFailures = _meter.CreateCounter<long>("msrc.summary.fetch.failures", "operations");
_summaryItemCount = _meter.CreateHistogram<long>("msrc.summary.items.count", "items");
_summaryWindowHours = _meter.CreateHistogram<double>("msrc.summary.window.hours", "hours");
_detailFetchAttempts = _meter.CreateCounter<long>("msrc.detail.fetch.attempts", "operations");
_detailFetchSuccess = _meter.CreateCounter<long>("msrc.detail.fetch.success", "operations");
_detailFetchNotModified = _meter.CreateCounter<long>("msrc.detail.fetch.not_modified", "operations");
_detailFetchFailures = _meter.CreateCounter<long>("msrc.detail.fetch.failures", "operations");
_detailEnqueued = _meter.CreateHistogram<long>("msrc.detail.enqueued.count", "documents");
_parseSuccess = _meter.CreateCounter<long>("msrc.parse.success", "documents");
_parseFailures = _meter.CreateCounter<long>("msrc.parse.failures", "documents");
_parseProductCount = _meter.CreateHistogram<long>("msrc.parse.products.count", "products");
_parseKbCount = _meter.CreateHistogram<long>("msrc.parse.kb.count", "kb");
_mapSuccess = _meter.CreateCounter<long>("msrc.map.success", "advisories");
_mapFailures = _meter.CreateCounter<long>("msrc.map.failures", "advisories");
_mapAliasCount = _meter.CreateHistogram<long>("msrc.map.aliases.count", "aliases");
_mapAffectedCount = _meter.CreateHistogram<long>("msrc.map.affected.count", "packages");
}
public void SummaryFetchAttempt() => _summaryFetchAttempts.Add(1);
public void SummaryFetchSuccess(int count, double? windowHours)
{
_summaryFetchSuccess.Add(1);
if (count >= 0)
{
_summaryItemCount.Record(count);
}
if (windowHours is { } value && value >= 0)
{
_summaryWindowHours.Record(value);
}
}
public void SummaryFetchFailure(string reason)
=> _summaryFetchFailures.Add(1, ReasonTag(reason));
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
public void DetailFetchFailure(string reason)
=> _detailFetchFailures.Add(1, ReasonTag(reason));
public void DetailEnqueued(int count)
{
if (count >= 0)
{
_detailEnqueued.Record(count);
}
}
public void ParseSuccess(int productCount, int kbCount)
{
_parseSuccess.Add(1);
if (productCount >= 0)
{
_parseProductCount.Record(productCount);
}
if (kbCount >= 0)
{
_parseKbCount.Record(kbCount);
}
}
public void ParseFailure(string reason)
=> _parseFailures.Add(1, ReasonTag(reason));
public void MapSuccess(int aliasCount, int packageCount)
{
_mapSuccess.Add(1);
if (aliasCount >= 0)
{
_mapAliasCount.Record(aliasCount);
}
if (packageCount >= 0)
{
_mapAffectedCount.Record(packageCount);
}
}
public void MapFailure(string reason)
=> _mapFailures.Add(1, ReasonTag(reason));
private static KeyValuePair<string, object?> ReasonTag(string reason)
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

View File

@@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
internal static class MsrcDocumentMetadata
{
public static Dictionary<string, string> CreateMetadata(MsrcVulnerabilitySummary summary)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["msrc.vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id,
["msrc.id"] = summary.Id,
};
if (summary.LastModifiedDate.HasValue)
{
metadata["msrc.lastModified"] = summary.LastModifiedDate.Value.ToString("O");
}
if (summary.ReleaseDate.HasValue)
{
metadata["msrc.releaseDate"] = summary.ReleaseDate.Value.ToString("O");
}
if (!string.IsNullOrWhiteSpace(summary.CvrfUrl))
{
metadata["msrc.cvrfUrl"] = summary.CvrfUrl!;
}
if (summary.CveNumbers.Count > 0)
{
metadata["msrc.cves"] = string.Join(",", summary.CveNumbers);
}
return metadata;
}
public static Dictionary<string, string> CreateCvrfMetadata(MsrcVulnerabilitySummary summary)
{
var metadata = CreateMetadata(summary);
metadata["msrc.cvrf"] = "true";
return metadata;
}
}

View File

@@ -0,0 +1,239 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Mongo.Documents;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
internal static class MsrcMapper
{
public static Advisory Map(MsrcAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
{
ArgumentNullException.ThrowIfNull(dto);
ArgumentNullException.ThrowIfNull(document);
var advisoryKey = dto.AdvisoryId;
var aliases = BuildAliases(dto);
var references = BuildReferences(dto, recordedAt);
var affectedPackages = BuildPackages(dto, recordedAt);
var cvssMetrics = BuildCvss(dto, recordedAt);
var provenance = new AdvisoryProvenance(
source: MsrcConnectorPlugin.SourceName,
kind: "advisory",
value: advisoryKey,
recordedAt,
new[] { ProvenanceFieldMasks.Advisory });
return new Advisory(
advisoryKey: advisoryKey,
title: dto.Title,
summary: dto.Description,
language: "en",
published: dto.ReleaseDate,
modified: dto.LastModifiedDate,
severity: NormalizeSeverity(dto.Severity),
exploitKnown: false,
aliases: aliases,
references: references,
affectedPackages: affectedPackages,
cvssMetrics: cvssMetrics,
provenance: new[] { provenance });
}
private static IReadOnlyList<string> BuildAliases(MsrcAdvisoryDto dto)
{
var aliases = new List<string> { dto.AdvisoryId };
foreach (var cve in dto.CveIds)
{
if (!string.IsNullOrWhiteSpace(cve))
{
aliases.Add(cve);
}
}
foreach (var kb in dto.KbIds)
{
if (!string.IsNullOrWhiteSpace(kb))
{
aliases.Add(kb.StartsWith("KB", StringComparison.OrdinalIgnoreCase) ? kb : $"KB{kb}");
}
}
return aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<AdvisoryReference> BuildReferences(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
var references = new List<AdvisoryReference>();
if (!string.IsNullOrWhiteSpace(dto.ReleaseNoteUrl))
{
references.Add(CreateReference(dto.ReleaseNoteUrl!, "details", recordedAt));
}
if (!string.IsNullOrWhiteSpace(dto.CvrfUrl))
{
references.Add(CreateReference(dto.CvrfUrl!, "cvrf", recordedAt));
}
foreach (var remediation in dto.Remediations)
{
if (!string.IsNullOrWhiteSpace(remediation.Url))
{
references.Add(CreateReference(
remediation.Url!,
string.Equals(remediation.Type, "security update", StringComparison.OrdinalIgnoreCase) ? "remediation" : remediation.Type ?? "reference",
recordedAt,
remediation.Description));
}
}
return references
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static AdvisoryReference CreateReference(string url, string kind, DateTimeOffset recordedAt, string? summary = null)
=> new(
url,
kind: kind.ToLowerInvariant(),
sourceTag: "msrc",
summary: summary,
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"reference",
url,
recordedAt,
new[] { ProvenanceFieldMasks.References }));
private static IReadOnlyList<AffectedPackage> BuildPackages(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.Products.Count == 0)
{
return Array.Empty<AffectedPackage>();
}
var packages = new List<AffectedPackage>(dto.Products.Count);
foreach (var product in dto.Products)
{
var identifier = string.IsNullOrWhiteSpace(product.Identifier) ? "Unknown Product" : product.Identifier;
var provenance = new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"package",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.AffectedPackages });
var notes = new List<string>();
if (!string.IsNullOrWhiteSpace(product.Platform))
{
notes.Add($"platform:{product.Platform}");
}
if (!string.IsNullOrWhiteSpace(product.Architecture))
{
notes.Add($"arch:{product.Architecture}");
}
if (!string.IsNullOrWhiteSpace(product.Cpe))
{
notes.Add($"cpe:{product.Cpe}");
}
var range = !string.IsNullOrWhiteSpace(product.BuildNumber)
? new[]
{
new AffectedVersionRange(
rangeKind: "custom",
introducedVersion: null,
fixedVersion: null,
lastAffectedVersion: null,
rangeExpression: $"build:{product.BuildNumber}",
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"package-range",
identifier,
recordedAt,
new[] { ProvenanceFieldMasks.VersionRanges })),
}
: Array.Empty<AffectedVersionRange>();
var normalizedRules = !string.IsNullOrWhiteSpace(product.BuildNumber)
? new[]
{
new NormalizedVersionRule(
scheme: "msrc.build",
type: NormalizedVersionRuleTypes.Exact,
value: product.BuildNumber,
notes: string.Join(";", notes.Where(static n => !string.IsNullOrWhiteSpace(n))))
}
: Array.Empty<NormalizedVersionRule>();
packages.Add(new AffectedPackage(
type: AffectedPackageTypes.Vendor,
identifier: identifier,
platform: product.Platform,
versionRanges: range,
statuses: Array.Empty<AffectedPackageStatus>(),
provenance: new[] { provenance },
normalizedVersions: normalizedRules));
}
return packages
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private static IReadOnlyList<CvssMetric> BuildCvss(MsrcAdvisoryDto dto, DateTimeOffset recordedAt)
{
if (dto.CvssBaseScore is null || string.IsNullOrWhiteSpace(dto.CvssVector))
{
return Array.Empty<CvssMetric>();
}
var severity = CvssSeverityFromScore(dto.CvssBaseScore.Value);
return new[]
{
new CvssMetric(
version: "3.1",
vector: dto.CvssVector!,
baseScore: dto.CvssBaseScore.Value,
baseSeverity: severity,
provenance: new AdvisoryProvenance(
MsrcConnectorPlugin.SourceName,
"cvss",
dto.AdvisoryId,
recordedAt,
new[] { ProvenanceFieldMasks.CvssMetrics })),
};
}
private static string CvssSeverityFromScore(double score)
=> score switch
{
< 0 => "none",
< 4 => "low",
< 7 => "medium",
< 9 => "high",
_ => "critical",
};
private static string? NormalizeSeverity(string? severity)
{
if (string.IsNullOrWhiteSpace(severity))
{
return null;
}
return severity.Trim().ToLowerInvariant();
}
}

View File

@@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public sealed record MsrcSummaryResponse
{
[JsonPropertyName("value")]
public List<MsrcVulnerabilitySummary> Value { get; init; } = new();
[JsonPropertyName("@odata.nextLink")]
public string? NextLink { get; init; }
}
public sealed record MsrcVulnerabilitySummary
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("vulnerabilityId")]
public string? VulnerabilityId { get; init; }
[JsonPropertyName("cveNumber")]
public string? CveNumber { get; init; }
[JsonPropertyName("cveNumbers")]
public IReadOnlyList<string> CveNumbers { get; init; } = Array.Empty<string>();
[JsonPropertyName("title")]
public string? Title { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("releaseDate")]
public DateTimeOffset? ReleaseDate { get; init; }
[JsonPropertyName("lastModifiedDate")]
public DateTimeOffset? LastModifiedDate { get; init; }
[JsonPropertyName("severity")]
public string? Severity { get; init; }
[JsonPropertyName("cvrfUrl")]
public string? CvrfUrl { get; init; }
}

View File

@@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration;
namespace StellaOps.Concelier.Connector.Vndr.Msrc.Internal;
public interface IMsrcTokenProvider
{
Task<string> GetAccessTokenAsync(CancellationToken cancellationToken);
}
public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly MsrcOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<MsrcTokenProvider> _logger;
private readonly SemaphoreSlim _refreshLock = new(1, 1);
private AccessToken? _currentToken;
public MsrcTokenProvider(
IHttpClientFactory httpClientFactory,
IOptions<MsrcOptions> options,
TimeProvider? timeProvider,
ILogger<MsrcTokenProvider> logger)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<string> GetAccessTokenAsync(CancellationToken cancellationToken)
{
var token = _currentToken;
if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow()))
{
return token.Token;
}
await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
token = _currentToken;
if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow()))
{
return token.Token;
}
_logger.LogInformation("Requesting new MSRC access token");
var client = _httpClientFactory.CreateClient(MsrcOptions.TokenClientName);
var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri())
{
Content = new FormUrlEncodedContent(new Dictionary<string, string>
{
["client_id"] = _options.ClientId,
["client_secret"] = _options.ClientSecret,
["grant_type"] = "client_credentials",
["scope"] = _options.Scope,
}),
};
using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var payload = await response.Content.ReadFromJsonAsync<TokenResponse>(cancellationToken: cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException("AAD token response was null.");
var expiresAt = _timeProvider.GetUtcNow().AddSeconds(payload.ExpiresIn - 60);
_currentToken = new AccessToken(payload.AccessToken, expiresAt);
return payload.AccessToken;
}
finally
{
_refreshLock.Release();
}
}
private Uri BuildTokenUri()
=> new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token");
public void Dispose() => _refreshLock.Dispose();
private sealed record AccessToken(string Token, DateTimeOffset ExpiresAt)
{
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt;
}
private sealed record TokenResponse
{
[JsonPropertyName("access_token")]
public string AccessToken { get; init; } = string.Empty;
[JsonPropertyName("expires_in")]
public int ExpiresIn { get; init; }
}
}

View File

@@ -1,8 +1,8 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Net; using System.Net;
using System.Net.Http; using System.Net.Http;
using System.Text; using System.Text;
using FluentAssertions; using FluentAssertions;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@@ -11,12 +11,13 @@ using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Cisco.CSAF; using StellaOps.Excititor.Connectors.Cisco.CSAF;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Storage.Mongo;
using System.Collections.Immutable; using System.Collections.Immutable;
using System.IO.Abstractions.TestingHelpers; using System.IO.Abstractions.TestingHelpers;
using Xunit; using Xunit;
using System.Threading; using System.Threading;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors; namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors;
@@ -159,14 +160,14 @@ public sealed class CiscoCsafConnectorTests
{ {
public VexConnectorState? CurrentState { get; private set; } public VexConnectorState? CurrentState { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken) public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(CurrentState); => ValueTask.FromResult(CurrentState);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
CurrentState = state; CurrentState = state;
return ValueTask.CompletedTask; return ValueTask.CompletedTask;
} }
} }
private sealed class InMemoryRawSink : IVexRawDocumentSink private sealed class InMemoryRawSink : IVexRawDocumentSink

View File

@@ -10,11 +10,12 @@ using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.MSRC.CSAF; using StellaOps.Excititor.Connectors.MSRC.CSAF;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Storage.Mongo;
using Xunit; using Xunit;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Connectors; namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Connectors;
@@ -316,14 +317,14 @@ public sealed class MsrcCsafConnectorTests
{ {
public VexConnectorState? State { get; private set; } public VexConnectorState? State { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken) public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(State); => ValueTask.FromResult(State);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
State = state; State = state;
return ValueTask.CompletedTask; return ValueTask.CompletedTask;
} }
} }
private sealed class TestHttpMessageHandler : HttpMessageHandler private sealed class TestHttpMessageHandler : HttpMessageHandler

View File

@@ -15,11 +15,12 @@ using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Oracle.CSAF; using StellaOps.Excititor.Connectors.Oracle.CSAF;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Storage.Mongo;
using System.IO.Abstractions.TestingHelpers; using System.IO.Abstractions.TestingHelpers;
using Xunit; using Xunit;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Connectors; namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Connectors;
@@ -254,14 +255,14 @@ public sealed class OracleCsafConnectorTests
{ {
public VexConnectorState? State { get; private set; } public VexConnectorState? State { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken) public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(State); => ValueTask.FromResult(State);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
State = state; State = state;
return ValueTask.CompletedTask; return ValueTask.CompletedTask;
} }
} }
private sealed class InMemoryRawSink : IVexRawDocumentSink private sealed class InMemoryRawSink : IVexRawDocumentSink

View File

@@ -10,9 +10,10 @@ using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata; using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Storage.Mongo;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.Connectors; namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.Connectors;
@@ -258,20 +259,20 @@ public sealed class RedHatCsafConnectorTests
{ {
public VexConnectorState? State { get; private set; } public VexConnectorState? State { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken) public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase)) if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase))
{ {
return ValueTask.FromResult<VexConnectorState?>(State); return ValueTask.FromResult<VexConnectorState?>(State);
} }
return ValueTask.FromResult<VexConnectorState?>(null); return ValueTask.FromResult<VexConnectorState?>(null);
} }
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
State = state; State = state;
return ValueTask.CompletedTask; return ValueTask.CompletedTask;
} }
} }
} }

View File

@@ -14,9 +14,9 @@ using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
internal sealed class RancherHubEventClient public sealed class RancherHubEventClient
{ {
private readonly IHttpClientFactory _httpClientFactory; private readonly IHttpClientFactory _httpClientFactory;
private readonly RancherHubTokenProvider _tokenProvider; private readonly RancherHubTokenProvider _tokenProvider;

View File

@@ -1,21 +1,21 @@
using System; using System;
using System.Collections.Immutable; using System.Collections.Immutable;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
internal sealed record RancherHubEventRecord( public sealed record RancherHubEventRecord(
string RawJson, string RawJson,
string? Id, string? Id,
string? Type, string? Type,
string? Channel, string? Channel,
DateTimeOffset? PublishedAt, DateTimeOffset? PublishedAt,
Uri? DocumentUri, Uri? DocumentUri,
string? DocumentDigest, string? DocumentDigest,
string? DocumentFormat); string? DocumentFormat);
internal sealed record RancherHubEventBatch( public sealed record RancherHubEventBatch(
string? Cursor, string? Cursor,
string? NextCursor, string? NextCursor,
ImmutableArray<RancherHubEventRecord> Events, ImmutableArray<RancherHubEventRecord> Events,
bool FromOfflineSnapshot, bool FromOfflineSnapshot,
string RawPayload); string RawPayload);

View File

@@ -1,344 +1,345 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Immutable; using System.Collections.Immutable;
using System.Linq; using System.Linq;
using System.Net.Http; using System.Net.Http;
using System.Net.Http.Headers; using System.Net.Http.Headers;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Logging; using System.Runtime.CompilerServices;
using StellaOps.Excititor.Connectors.Abstractions; using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub;
public sealed class RancherHubConnector : VexConnectorBase
{ public sealed class RancherHubConnector : VexConnectorBase
private static readonly VexConnectorDescriptor StaticDescriptor = new( {
id: "excititor:suse.rancher", private static readonly VexConnectorDescriptor StaticDescriptor = new(
kind: VexProviderKind.Hub, id: "excititor:suse.rancher",
displayName: "SUSE Rancher VEX Hub") kind: VexProviderKind.Hub,
{ displayName: "SUSE Rancher VEX Hub")
Tags = ImmutableArray.Create("hub", "suse", "offline"), {
}; Tags = ImmutableArray.Create("hub", "suse", "offline"),
};
private readonly RancherHubMetadataLoader _metadataLoader;
private readonly RancherHubEventClient _eventClient; private readonly RancherHubMetadataLoader _metadataLoader;
private readonly RancherHubCheckpointManager _checkpointManager; private readonly RancherHubEventClient _eventClient;
private readonly RancherHubTokenProvider _tokenProvider; private readonly RancherHubCheckpointManager _checkpointManager;
private readonly IHttpClientFactory _httpClientFactory; private readonly RancherHubTokenProvider _tokenProvider;
private readonly IEnumerable<IVexConnectorOptionsValidator<RancherHubConnectorOptions>> _validators; private readonly IHttpClientFactory _httpClientFactory;
private readonly IEnumerable<IVexConnectorOptionsValidator<RancherHubConnectorOptions>> _validators;
private RancherHubConnectorOptions? _options;
private RancherHubMetadataResult? _metadata; private RancherHubConnectorOptions? _options;
private RancherHubMetadataResult? _metadata;
public RancherHubConnector(
RancherHubMetadataLoader metadataLoader, public RancherHubConnector(
RancherHubEventClient eventClient, RancherHubMetadataLoader metadataLoader,
RancherHubCheckpointManager checkpointManager, RancherHubEventClient eventClient,
RancherHubTokenProvider tokenProvider, RancherHubCheckpointManager checkpointManager,
IHttpClientFactory httpClientFactory, RancherHubTokenProvider tokenProvider,
ILogger<RancherHubConnector> logger, IHttpClientFactory httpClientFactory,
TimeProvider timeProvider, ILogger<RancherHubConnector> logger,
IEnumerable<IVexConnectorOptionsValidator<RancherHubConnectorOptions>>? validators = null) TimeProvider timeProvider,
: base(StaticDescriptor, logger, timeProvider) IEnumerable<IVexConnectorOptionsValidator<RancherHubConnectorOptions>>? validators = null)
{ : base(StaticDescriptor, logger, timeProvider)
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); {
_eventClient = eventClient ?? throw new ArgumentNullException(nameof(eventClient)); _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
_checkpointManager = checkpointManager ?? throw new ArgumentNullException(nameof(checkpointManager)); _eventClient = eventClient ?? throw new ArgumentNullException(nameof(eventClient));
_tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); _checkpointManager = checkpointManager ?? throw new ArgumentNullException(nameof(checkpointManager));
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider));
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<RancherHubConnectorOptions>>(); _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
} _validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<RancherHubConnectorOptions>>();
}
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
{ public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
_options = VexConnectorOptionsBinder.Bind( {
Descriptor, _options = VexConnectorOptionsBinder.Bind(
settings, Descriptor,
validators: _validators); settings,
validators: _validators);
_metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
_metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Information, "validate", "Rancher hub discovery loaded.", new Dictionary<string, object?>
{ LogConnectorEvent(LogLevel.Information, "validate", "Rancher hub discovery loaded.", new Dictionary<string, object?>
["discoveryUri"] = _options.DiscoveryUri.ToString(), {
["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), ["discoveryUri"] = _options.DiscoveryUri.ToString(),
["requiresAuth"] = _metadata.Metadata.Subscription.RequiresAuthentication, ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(),
["fromOffline"] = _metadata.FromOfflineSnapshot, ["requiresAuth"] = _metadata.Metadata.Subscription.RequiresAuthentication,
}); ["fromOffline"] = _metadata.FromOfflineSnapshot,
} });
}
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
{ public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
ArgumentNullException.ThrowIfNull(context); {
ArgumentNullException.ThrowIfNull(context);
if (_options is null)
{ if (_options is null)
throw new InvalidOperationException("Connector must be validated before fetch operations."); {
} throw new InvalidOperationException("Connector must be validated before fetch operations.");
}
if (_metadata is null)
{ if (_metadata is null)
_metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); {
} _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
}
var checkpoint = await _checkpointManager.LoadAsync(Descriptor.Id, context, cancellationToken).ConfigureAwait(false);
var digestHistory = checkpoint.Digests.ToList(); var checkpoint = await _checkpointManager.LoadAsync(Descriptor.Id, context, cancellationToken).ConfigureAwait(false);
var dedupeSet = new HashSet<string>(checkpoint.Digests, StringComparer.OrdinalIgnoreCase); var digestHistory = checkpoint.Digests.ToList();
var latestCursor = checkpoint.Cursor; var dedupeSet = new HashSet<string>(checkpoint.Digests, StringComparer.OrdinalIgnoreCase);
var latestPublishedAt = checkpoint.LastPublishedAt ?? checkpoint.EffectiveSince; var latestCursor = checkpoint.Cursor;
var stateChanged = false; var latestPublishedAt = checkpoint.LastPublishedAt ?? checkpoint.EffectiveSince;
var stateChanged = false;
LogConnectorEvent(LogLevel.Information, "fetch_start", "Starting Rancher hub event ingestion.", new Dictionary<string, object?>
{ LogConnectorEvent(LogLevel.Information, "fetch_start", "Starting Rancher hub event ingestion.", new Dictionary<string, object?>
["since"] = checkpoint.EffectiveSince?.ToString("O"), {
["cursor"] = checkpoint.Cursor, ["since"] = checkpoint.EffectiveSince?.ToString("O"),
["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), ["cursor"] = checkpoint.Cursor,
["offline"] = checkpoint.Cursor is null && _options.PreferOfflineSnapshot, ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(),
}); ["offline"] = checkpoint.Cursor is null && _options.PreferOfflineSnapshot,
});
await foreach (var batch in _eventClient.FetchEventBatchesAsync(
_options, await foreach (var batch in _eventClient.FetchEventBatchesAsync(
_metadata.Metadata, _options,
checkpoint.Cursor, _metadata.Metadata,
checkpoint.EffectiveSince, checkpoint.Cursor,
_metadata.Metadata.Subscription.Channels, checkpoint.EffectiveSince,
cancellationToken).ConfigureAwait(false)) _metadata.Metadata.Subscription.Channels,
{ cancellationToken).ConfigureAwait(false))
LogConnectorEvent(LogLevel.Debug, "batch", "Processing Rancher hub batch.", new Dictionary<string, object?> {
{ LogConnectorEvent(LogLevel.Debug, "batch", "Processing Rancher hub batch.", new Dictionary<string, object?>
["cursor"] = batch.Cursor, {
["nextCursor"] = batch.NextCursor, ["cursor"] = batch.Cursor,
["count"] = batch.Events.Length, ["nextCursor"] = batch.NextCursor,
["offline"] = batch.FromOfflineSnapshot, ["count"] = batch.Events.Length,
}); ["offline"] = batch.FromOfflineSnapshot,
});
if (!string.IsNullOrWhiteSpace(batch.NextCursor) && !string.Equals(batch.NextCursor, latestCursor, StringComparison.Ordinal))
{ if (!string.IsNullOrWhiteSpace(batch.NextCursor) && !string.Equals(batch.NextCursor, latestCursor, StringComparison.Ordinal))
latestCursor = batch.NextCursor; {
stateChanged = true; latestCursor = batch.NextCursor;
} stateChanged = true;
else if (string.IsNullOrWhiteSpace(latestCursor) && !string.IsNullOrWhiteSpace(batch.Cursor)) }
{ else if (string.IsNullOrWhiteSpace(latestCursor) && !string.IsNullOrWhiteSpace(batch.Cursor))
latestCursor = batch.Cursor; {
} latestCursor = batch.Cursor;
}
foreach (var record in batch.Events)
{ foreach (var record in batch.Events)
cancellationToken.ThrowIfCancellationRequested(); {
cancellationToken.ThrowIfCancellationRequested();
var result = await ProcessEventAsync(record, batch, context, dedupeSet, digestHistory, cancellationToken).ConfigureAwait(false);
if (result.ProcessedDocument is not null) var result = await ProcessEventAsync(record, batch, context, dedupeSet, digestHistory, cancellationToken).ConfigureAwait(false);
{ if (result.ProcessedDocument is not null)
yield return result.ProcessedDocument; {
stateChanged = true; yield return result.ProcessedDocument;
if (result.PublishedAt is { } published && (latestPublishedAt is null || published > latestPublishedAt)) stateChanged = true;
{ if (result.PublishedAt is { } published && (latestPublishedAt is null || published > latestPublishedAt))
latestPublishedAt = published; {
} latestPublishedAt = published;
} }
else if (result.Quarantined) }
{ else if (result.Quarantined)
stateChanged = true; {
} stateChanged = true;
} }
} }
}
if (stateChanged || !string.Equals(latestCursor, checkpoint.Cursor, StringComparison.Ordinal) || latestPublishedAt != checkpoint.LastPublishedAt)
{ if (stateChanged || !string.Equals(latestCursor, checkpoint.Cursor, StringComparison.Ordinal) || latestPublishedAt != checkpoint.LastPublishedAt)
await _checkpointManager.SaveAsync( {
Descriptor.Id, await _checkpointManager.SaveAsync(
latestCursor, Descriptor.Id,
latestPublishedAt, latestCursor,
digestHistory.ToImmutableArray(), latestPublishedAt,
cancellationToken).ConfigureAwait(false); digestHistory.ToImmutableArray(),
} cancellationToken).ConfigureAwait(false);
} }
}
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("RancherHubConnector relies on format-specific normalizers for CSAF/OpenVEX payloads."); public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
=> throw new NotSupportedException("RancherHubConnector relies on format-specific normalizers for CSAF/OpenVEX payloads.");
public RancherHubMetadata? GetCachedMetadata() => _metadata?.Metadata;
public RancherHubMetadata? GetCachedMetadata() => _metadata?.Metadata;
private async Task<EventProcessingResult> ProcessEventAsync(
RancherHubEventRecord record, private async Task<EventProcessingResult> ProcessEventAsync(
RancherHubEventBatch batch, RancherHubEventRecord record,
VexConnectorContext context, RancherHubEventBatch batch,
HashSet<string> dedupeSet, VexConnectorContext context,
List<string> digestHistory, HashSet<string> dedupeSet,
CancellationToken cancellationToken) List<string> digestHistory,
{ CancellationToken cancellationToken)
var quarantineKey = BuildQuarantineKey(record); {
if (dedupeSet.Contains(quarantineKey)) var quarantineKey = BuildQuarantineKey(record);
{ if (dedupeSet.Contains(quarantineKey))
return EventProcessingResult.QuarantinedOnly; {
} return EventProcessingResult.QuarantinedOnly;
}
if (record.DocumentUri is null || string.IsNullOrWhiteSpace(record.Id))
{ if (record.DocumentUri is null || string.IsNullOrWhiteSpace(record.Id))
await QuarantineAsync(record, batch, "missing documentUri or id", context, cancellationToken).ConfigureAwait(false); {
AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); await QuarantineAsync(record, batch, "missing documentUri or id", context, cancellationToken).ConfigureAwait(false);
return EventProcessingResult.QuarantinedOnly; AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory);
} return EventProcessingResult.QuarantinedOnly;
}
var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName);
using var request = await CreateDocumentRequestAsync(record.DocumentUri, cancellationToken).ConfigureAwait(false); var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName);
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); using var request = await CreateDocumentRequestAsync(record.DocumentUri, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode) using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
{ if (!response.IsSuccessStatusCode)
await QuarantineAsync(record, batch, $"document fetch failed ({(int)response.StatusCode} {response.StatusCode})", context, cancellationToken).ConfigureAwait(false); {
AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); await QuarantineAsync(record, batch, $"document fetch failed ({(int)response.StatusCode} {response.StatusCode})", context, cancellationToken).ConfigureAwait(false);
return EventProcessingResult.QuarantinedOnly; AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory);
} return EventProcessingResult.QuarantinedOnly;
}
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var publishedAt = record.PublishedAt ?? UtcNow(); var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var metadata = BuildMetadata(builder => builder var publishedAt = record.PublishedAt ?? UtcNow();
.Add("rancher.event.id", record.Id) var metadata = BuildMetadata(builder => builder
.Add("rancher.event.type", record.Type) .Add("rancher.event.id", record.Id)
.Add("rancher.event.channel", record.Channel) .Add("rancher.event.type", record.Type)
.Add("rancher.event.published", publishedAt) .Add("rancher.event.channel", record.Channel)
.Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor) .Add("rancher.event.published", publishedAt)
.Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false") .Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor)
.Add("rancher.event.declaredDigest", record.DocumentDigest)); .Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false")
.Add("rancher.event.declaredDigest", record.DocumentDigest));
var format = ResolveFormat(record.DocumentFormat);
var document = CreateRawDocument(format, record.DocumentUri, contentBytes, metadata); var format = ResolveFormat(record.DocumentFormat);
var document = CreateRawDocument(format, record.DocumentUri, contentBytes, metadata);
if (!string.IsNullOrWhiteSpace(record.DocumentDigest))
{ if (!string.IsNullOrWhiteSpace(record.DocumentDigest))
var declared = NormalizeDigest(record.DocumentDigest); {
var computed = NormalizeDigest(document.Digest); var declared = NormalizeDigest(record.DocumentDigest);
if (!string.Equals(declared, computed, StringComparison.OrdinalIgnoreCase)) var computed = NormalizeDigest(document.Digest);
{ if (!string.Equals(declared, computed, StringComparison.OrdinalIgnoreCase))
await QuarantineAsync(record, batch, $"digest mismatch (declared {record.DocumentDigest}, computed {document.Digest})", context, cancellationToken).ConfigureAwait(false); {
AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); await QuarantineAsync(record, batch, $"digest mismatch (declared {record.DocumentDigest}, computed {document.Digest})", context, cancellationToken).ConfigureAwait(false);
return EventProcessingResult.QuarantinedOnly; AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory);
} return EventProcessingResult.QuarantinedOnly;
} }
}
if (!dedupeSet.Add(document.Digest))
{ if (!dedupeSet.Add(document.Digest))
return EventProcessingResult.Skipped; {
} return EventProcessingResult.Skipped;
}
digestHistory.Add(document.Digest);
await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); digestHistory.Add(document.Digest);
return new EventProcessingResult(document, false, publishedAt); await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false);
} return new EventProcessingResult(document, false, publishedAt);
}
private async Task<HttpRequestMessage> CreateDocumentRequestAsync(Uri documentUri, CancellationToken cancellationToken)
{ private async Task<HttpRequestMessage> CreateDocumentRequestAsync(Uri documentUri, CancellationToken cancellationToken)
var request = new HttpRequestMessage(HttpMethod.Get, documentUri); {
if (_metadata?.Metadata.Subscription.RequiresAuthentication ?? false) var request = new HttpRequestMessage(HttpMethod.Get, documentUri);
{ if (_metadata?.Metadata.Subscription.RequiresAuthentication ?? false)
var token = await _tokenProvider.GetAccessTokenAsync(_options!, cancellationToken).ConfigureAwait(false); {
if (token is not null) var token = await _tokenProvider.GetAccessTokenAsync(_options!, cancellationToken).ConfigureAwait(false);
{ if (token is not null)
var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType; {
request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value); var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType;
} request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value);
} }
}
return request;
} return request;
}
private async Task QuarantineAsync(
RancherHubEventRecord record, private async Task QuarantineAsync(
RancherHubEventBatch batch, RancherHubEventRecord record,
string reason, RancherHubEventBatch batch,
VexConnectorContext context, string reason,
CancellationToken cancellationToken) VexConnectorContext context,
{ CancellationToken cancellationToken)
var metadata = BuildMetadata(builder => builder {
.Add("rancher.event.id", record.Id) var metadata = BuildMetadata(builder => builder
.Add("rancher.event.type", record.Type) .Add("rancher.event.id", record.Id)
.Add("rancher.event.channel", record.Channel) .Add("rancher.event.type", record.Type)
.Add("rancher.event.quarantine", "true") .Add("rancher.event.channel", record.Channel)
.Add("rancher.event.error", reason) .Add("rancher.event.quarantine", "true")
.Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor) .Add("rancher.event.error", reason)
.Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false")); .Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor)
.Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false"));
var sourceUri = record.DocumentUri ?? _metadata?.Metadata.Subscription.EventsUri ?? _options!.DiscoveryUri;
var payload = Encoding.UTF8.GetBytes(record.RawJson); var sourceUri = record.DocumentUri ?? _metadata?.Metadata.Subscription.EventsUri ?? _options!.DiscoveryUri;
var document = CreateRawDocument(VexDocumentFormat.Csaf, sourceUri, payload, metadata); var payload = Encoding.UTF8.GetBytes(record.RawJson);
await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); var document = CreateRawDocument(VexDocumentFormat.Csaf, sourceUri, payload, metadata);
await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false);
LogConnectorEvent(LogLevel.Warning, "quarantine", "Rancher hub event moved to quarantine.", new Dictionary<string, object?>
{ LogConnectorEvent(LogLevel.Warning, "quarantine", "Rancher hub event moved to quarantine.", new Dictionary<string, object?>
["eventId"] = record.Id ?? "(missing)", {
["reason"] = reason, ["eventId"] = record.Id ?? "(missing)",
}); ["reason"] = reason,
} });
}
private static void AddQuarantineDigest(string key, HashSet<string> dedupeSet, List<string> digestHistory)
{ private static void AddQuarantineDigest(string key, HashSet<string> dedupeSet, List<string> digestHistory)
if (dedupeSet.Add(key)) {
{ if (dedupeSet.Add(key))
digestHistory.Add(key); {
} digestHistory.Add(key);
} }
}
private static string BuildQuarantineKey(RancherHubEventRecord record)
{ private static string BuildQuarantineKey(RancherHubEventRecord record)
if (!string.IsNullOrWhiteSpace(record.Id)) {
{ if (!string.IsNullOrWhiteSpace(record.Id))
return $"quarantine:{record.Id}"; {
} return $"quarantine:{record.Id}";
}
Span<byte> hash = stackalloc byte[32];
var bytes = Encoding.UTF8.GetBytes(record.RawJson); Span<byte> hash = stackalloc byte[32];
if (!SHA256.TryHashData(bytes, hash, out _)) var bytes = Encoding.UTF8.GetBytes(record.RawJson);
{ if (!SHA256.TryHashData(bytes, hash, out _))
using var sha = SHA256.Create(); {
hash = sha.ComputeHash(bytes); using var sha = SHA256.Create();
} hash = sha.ComputeHash(bytes);
}
return $"quarantine:{Convert.ToHexString(hash).ToLowerInvariant()}";
} return $"quarantine:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static string NormalizeDigest(string digest)
{ private static string NormalizeDigest(string digest)
if (string.IsNullOrWhiteSpace(digest)) {
{ if (string.IsNullOrWhiteSpace(digest))
return digest; {
} return digest;
}
var trimmed = digest.Trim();
return trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) var trimmed = digest.Trim();
? trimmed.ToLowerInvariant() return trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
: $"sha256:{trimmed.ToLowerInvariant()}"; ? trimmed.ToLowerInvariant()
} : $"sha256:{trimmed.ToLowerInvariant()}";
}
private static VexDocumentFormat ResolveFormat(string? format)
{ private static VexDocumentFormat ResolveFormat(string? format)
if (string.IsNullOrWhiteSpace(format)) {
{ if (string.IsNullOrWhiteSpace(format))
return VexDocumentFormat.Csaf; {
} return VexDocumentFormat.Csaf;
}
return format.ToLowerInvariant() switch
{ return format.ToLowerInvariant() switch
"csaf" or "csaf_json" or "json" => VexDocumentFormat.Csaf, {
"cyclonedx" or "cyclonedx_vex" => VexDocumentFormat.CycloneDx, "csaf" or "csaf_json" or "json" => VexDocumentFormat.Csaf,
"openvex" => VexDocumentFormat.OpenVex, "cyclonedx" or "cyclonedx_vex" => VexDocumentFormat.CycloneDx,
"oci" or "oci_attestation" or "attestation" => VexDocumentFormat.OciAttestation, "openvex" => VexDocumentFormat.OpenVex,
_ => VexDocumentFormat.Csaf, "oci" or "oci_attestation" or "attestation" => VexDocumentFormat.OciAttestation,
}; _ => VexDocumentFormat.Csaf,
} };
}
private sealed record EventProcessingResult(VexRawDocument? ProcessedDocument, bool Quarantined, DateTimeOffset? PublishedAt)
{ private sealed record EventProcessingResult(VexRawDocument? ProcessedDocument, bool Quarantined, DateTimeOffset? PublishedAt)
public static EventProcessingResult QuarantinedOnly { get; } = new(null, true, null); {
public static EventProcessingResult QuarantinedOnly { get; } = new(null, true, null);
public static EventProcessingResult Skipped { get; } = new(null, false, null);
} public static EventProcessingResult Skipped { get; } = new(null, false, null);
} }
}

View File

@@ -6,15 +6,15 @@ using System.Threading.Tasks;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State; namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
internal sealed record RancherHubCheckpointState( public sealed record RancherHubCheckpointState(
string? Cursor, string? Cursor,
DateTimeOffset? LastPublishedAt, DateTimeOffset? LastPublishedAt,
DateTimeOffset? EffectiveSince, DateTimeOffset? EffectiveSince,
ImmutableArray<string> Digests); ImmutableArray<string> Digests);
internal sealed class RancherHubCheckpointManager public sealed class RancherHubCheckpointManager
{ {
private const string CheckpointPrefix = "checkpoint:"; private const string CheckpointPrefix = "checkpoint:";
private readonly IVexConnectorStateRepository _repository; private readonly IVexConnectorStateRepository _repository;

View File

@@ -1,309 +1,310 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Immutable; using System.Collections.Immutable;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Net.Http; using System.Net.Http;
using System.Net.Http.Headers; using System.Net.Http.Headers;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Threading; using System.Threading;
using FluentAssertions; using FluentAssertions;
using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Excititor.Connectors.Abstractions; using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF; using StellaOps.Excititor.Connectors.Ubuntu.CSAF;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Storage.Mongo;
using System.IO.Abstractions.TestingHelpers; using System.IO.Abstractions.TestingHelpers;
using Xunit; using Xunit;
using MongoDB.Driver;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors;
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors;
public sealed class UbuntuCsafConnectorTests
{ public sealed class UbuntuCsafConnectorTests
[Fact] {
public async Task FetchAsync_IngestsNewDocument_UpdatesStateAndUsesEtag() [Fact]
{ public async Task FetchAsync_IngestsNewDocument_UpdatesStateAndUsesEtag()
var baseUri = new Uri("https://ubuntu.test/security/csaf/"); {
var indexUri = new Uri(baseUri, "index.json"); var baseUri = new Uri("https://ubuntu.test/security/csaf/");
var catalogUri = new Uri(baseUri, "stable/catalog.json"); var indexUri = new Uri(baseUri, "index.json");
var advisoryUri = new Uri(baseUri, "stable/USN-2025-0001.json"); var catalogUri = new Uri(baseUri, "stable/catalog.json");
var advisoryUri = new Uri(baseUri, "stable/USN-2025-0001.json");
var manifest = CreateTestManifest(advisoryUri, "USN-2025-0001", "2025-10-18T00:00:00Z");
var documentPayload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); var manifest = CreateTestManifest(advisoryUri, "USN-2025-0001", "2025-10-18T00:00:00Z");
var documentSha = ComputeSha256(documentPayload); var documentPayload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}");
var documentSha = ComputeSha256(documentPayload);
var indexJson = manifest.IndexJson;
var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", documentSha, StringComparison.Ordinal); var indexJson = manifest.IndexJson;
var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, documentPayload, expectedEtag: "etag-123"); var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", documentSha, StringComparison.Ordinal);
var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, documentPayload, expectedEtag: "etag-123");
var httpClient = new HttpClient(handler);
var httpFactory = new SingleClientFactory(httpClient); var httpClient = new HttpClient(handler);
var cache = new MemoryCache(new MemoryCacheOptions()); var httpFactory = new SingleClientFactory(httpClient);
var fileSystem = new MockFileSystem(); var cache = new MemoryCache(new MemoryCacheOptions());
var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger<UbuntuCatalogLoader>.Instance, TimeProvider.System); var fileSystem = new MockFileSystem();
var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger<UbuntuCatalogLoader>.Instance, TimeProvider.System);
var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem);
var stateRepository = new InMemoryConnectorStateRepository(); var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem);
var connector = new UbuntuCsafConnector( var stateRepository = new InMemoryConnectorStateRepository();
loader, var connector = new UbuntuCsafConnector(
httpFactory, loader,
stateRepository, httpFactory,
new[] { optionsValidator }, stateRepository,
NullLogger<UbuntuCsafConnector>.Instance, new[] { optionsValidator },
TimeProvider.System); NullLogger<UbuntuCsafConnector>.Instance,
TimeProvider.System);
var settings = new VexConnectorSettings(ImmutableDictionary<string, string>.Empty);
await connector.ValidateAsync(settings, CancellationToken.None); var settings = new VexConnectorSettings(ImmutableDictionary<string, string>.Empty);
await connector.ValidateAsync(settings, CancellationToken.None);
var sink = new InMemoryRawSink();
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); var sink = new InMemoryRawSink();
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider());
var documents = new List<VexRawDocument>();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) var documents = new List<VexRawDocument>();
{ await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
documents.Add(doc); {
} documents.Add(doc);
}
documents.Should().HaveCount(1);
sink.Documents.Should().HaveCount(1); documents.Should().HaveCount(1);
var stored = sink.Documents.Single(); sink.Documents.Should().HaveCount(1);
stored.Digest.Should().Be($"sha256:{documentSha}"); var stored = sink.Documents.Single();
stored.Metadata.TryGetValue("ubuntu.etag", out var storedEtag).Should().BeTrue(); stored.Digest.Should().Be($"sha256:{documentSha}");
storedEtag.Should().Be("etag-123"); stored.Metadata.TryGetValue("ubuntu.etag", out var storedEtag).Should().BeTrue();
storedEtag.Should().Be("etag-123");
stateRepository.CurrentState.Should().NotBeNull();
stateRepository.CurrentState!.DocumentDigests.Should().Contain($"sha256:{documentSha}"); stateRepository.CurrentState.Should().NotBeNull();
stateRepository.CurrentState.DocumentDigests.Should().Contain($"etag:{advisoryUri}|etag-123"); stateRepository.CurrentState!.DocumentDigests.Should().Contain($"sha256:{documentSha}");
stateRepository.CurrentState.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-18T00:00:00Z")); stateRepository.CurrentState.DocumentDigests.Should().Contain($"etag:{advisoryUri}|etag-123");
stateRepository.CurrentState.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-18T00:00:00Z"));
handler.DocumentRequestCount.Should().Be(1);
handler.DocumentRequestCount.Should().Be(1);
// Second run: Expect connector to send If-None-Match and skip download via 304.
sink.Documents.Clear(); // Second run: Expect connector to send If-None-Match and skip download via 304.
documents.Clear(); sink.Documents.Clear();
documents.Clear();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
{ await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
documents.Add(doc); {
} documents.Add(doc);
}
documents.Should().BeEmpty();
sink.Documents.Should().BeEmpty(); documents.Should().BeEmpty();
handler.DocumentRequestCount.Should().Be(2); sink.Documents.Should().BeEmpty();
handler.SeenIfNoneMatch.Should().Contain("\"etag-123\""); handler.DocumentRequestCount.Should().Be(2);
} handler.SeenIfNoneMatch.Should().Contain("\"etag-123\"");
}
[Fact]
public async Task FetchAsync_SkipsWhenChecksumMismatch() [Fact]
{ public async Task FetchAsync_SkipsWhenChecksumMismatch()
var baseUri = new Uri("https://ubuntu.test/security/csaf/"); {
var indexUri = new Uri(baseUri, "index.json"); var baseUri = new Uri("https://ubuntu.test/security/csaf/");
var catalogUri = new Uri(baseUri, "stable/catalog.json"); var indexUri = new Uri(baseUri, "index.json");
var advisoryUri = new Uri(baseUri, "stable/USN-2025-0002.json"); var catalogUri = new Uri(baseUri, "stable/catalog.json");
var advisoryUri = new Uri(baseUri, "stable/USN-2025-0002.json");
var manifest = CreateTestManifest(advisoryUri, "USN-2025-0002", "2025-10-18T00:00:00Z");
var indexJson = manifest.IndexJson; var manifest = CreateTestManifest(advisoryUri, "USN-2025-0002", "2025-10-18T00:00:00Z");
var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", new string('a', 64), StringComparison.Ordinal); var indexJson = manifest.IndexJson;
var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"), expectedEtag: "etag-999"); var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", new string('a', 64), StringComparison.Ordinal);
var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"), expectedEtag: "etag-999");
var httpClient = new HttpClient(handler);
var httpFactory = new SingleClientFactory(httpClient); var httpClient = new HttpClient(handler);
var cache = new MemoryCache(new MemoryCacheOptions()); var httpFactory = new SingleClientFactory(httpClient);
var fileSystem = new MockFileSystem(); var cache = new MemoryCache(new MemoryCacheOptions());
var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger<UbuntuCatalogLoader>.Instance, TimeProvider.System); var fileSystem = new MockFileSystem();
var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem); var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger<UbuntuCatalogLoader>.Instance, TimeProvider.System);
var stateRepository = new InMemoryConnectorStateRepository(); var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem);
var stateRepository = new InMemoryConnectorStateRepository();
var connector = new UbuntuCsafConnector(
loader, var connector = new UbuntuCsafConnector(
httpFactory, loader,
stateRepository, httpFactory,
new[] { optionsValidator }, stateRepository,
NullLogger<UbuntuCsafConnector>.Instance, new[] { optionsValidator },
TimeProvider.System); NullLogger<UbuntuCsafConnector>.Instance,
TimeProvider.System);
await connector.ValidateAsync(new VexConnectorSettings(ImmutableDictionary<string, string>.Empty), CancellationToken.None);
await connector.ValidateAsync(new VexConnectorSettings(ImmutableDictionary<string, string>.Empty), CancellationToken.None);
var sink = new InMemoryRawSink();
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); var sink = new InMemoryRawSink();
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider());
var documents = new List<VexRawDocument>();
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) var documents = new List<VexRawDocument>();
{ await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
documents.Add(doc); {
} documents.Add(doc);
}
documents.Should().BeEmpty();
sink.Documents.Should().BeEmpty(); documents.Should().BeEmpty();
stateRepository.CurrentState.Should().NotBeNull(); sink.Documents.Should().BeEmpty();
stateRepository.CurrentState!.DocumentDigests.Should().BeEmpty(); stateRepository.CurrentState.Should().NotBeNull();
handler.DocumentRequestCount.Should().Be(1); stateRepository.CurrentState!.DocumentDigests.Should().BeEmpty();
} handler.DocumentRequestCount.Should().Be(1);
}
private static (string IndexJson, string CatalogJson) CreateTestManifest(Uri advisoryUri, string advisoryId, string timestamp)
{ private static (string IndexJson, string CatalogJson) CreateTestManifest(Uri advisoryUri, string advisoryId, string timestamp)
var indexJson = $$""" {
{ var indexJson = """
"generated": "2025-10-18T00:00:00Z", {
"channels": [ "generated": "2025-10-18T00:00:00Z",
{ "channels": [
"name": "stable", {
"catalogUrl": "{{advisoryUri.GetLeftPart(UriPartial.Authority)}}/security/csaf/stable/catalog.json", "name": "stable",
"sha256": "ignore" "catalogUrl": "{{advisoryUri.GetLeftPart(UriPartial.Authority)}}/security/csaf/stable/catalog.json",
} "sha256": "ignore"
] }
} ]
"""; }
""";
var catalogJson = $$"""
{ var catalogJson = """
"resources": [ {
{ "resources": [
"id": "{{advisoryId}}", {
"type": "csaf", "id": "{{advisoryId}}",
"url": "{{advisoryUri}}", "type": "csaf",
"last_modified": "{{timestamp}}", "url": "{{advisoryUri}}",
"hashes": { "last_modified": "{{timestamp}}",
"sha256": "{{SHA256}}" "hashes": {
}, "sha256": "{{SHA256}}"
"etag": "\"etag-123\"", },
"title": "{{advisoryId}}" "etag": "\"etag-123\"",
} "title": "{{advisoryId}}"
] }
} ]
"""; }
""";
return (indexJson, catalogJson);
} return (indexJson, catalogJson);
}
private static string ComputeSha256(ReadOnlySpan<byte> payload)
{ private static string ComputeSha256(ReadOnlySpan<byte> payload)
Span<byte> buffer = stackalloc byte[32]; {
SHA256.HashData(payload, buffer); Span<byte> buffer = stackalloc byte[32];
return Convert.ToHexString(buffer).ToLowerInvariant(); SHA256.HashData(payload, buffer);
} return Convert.ToHexString(buffer).ToLowerInvariant();
}
private sealed class SingleClientFactory : IHttpClientFactory
{ private sealed class SingleClientFactory : IHttpClientFactory
private readonly HttpClient _client; {
private readonly HttpClient _client;
public SingleClientFactory(HttpClient client)
{ public SingleClientFactory(HttpClient client)
_client = client; {
} _client = client;
}
public HttpClient CreateClient(string name) => _client;
} public HttpClient CreateClient(string name) => _client;
}
private sealed class UbuntuTestHttpHandler : HttpMessageHandler
{ private sealed class UbuntuTestHttpHandler : HttpMessageHandler
private readonly Uri _indexUri; {
private readonly string _indexPayload; private readonly Uri _indexUri;
private readonly Uri _catalogUri; private readonly string _indexPayload;
private readonly string _catalogPayload; private readonly Uri _catalogUri;
private readonly Uri _documentUri; private readonly string _catalogPayload;
private readonly byte[] _documentPayload; private readonly Uri _documentUri;
private readonly string _expectedEtag; private readonly byte[] _documentPayload;
private readonly string _expectedEtag;
public int DocumentRequestCount { get; private set; }
public List<string> SeenIfNoneMatch { get; } = new(); public int DocumentRequestCount { get; private set; }
public List<string> SeenIfNoneMatch { get; } = new();
public UbuntuTestHttpHandler(Uri indexUri, string indexPayload, Uri catalogUri, string catalogPayload, Uri documentUri, byte[] documentPayload, string expectedEtag)
{ public UbuntuTestHttpHandler(Uri indexUri, string indexPayload, Uri catalogUri, string catalogPayload, Uri documentUri, byte[] documentPayload, string expectedEtag)
_indexUri = indexUri; {
_indexPayload = indexPayload; _indexUri = indexUri;
_catalogUri = catalogUri; _indexPayload = indexPayload;
_catalogPayload = catalogPayload; _catalogUri = catalogUri;
_documentUri = documentUri; _catalogPayload = catalogPayload;
_documentPayload = documentPayload; _documentUri = documentUri;
_expectedEtag = expectedEtag; _documentPayload = documentPayload;
} _expectedEtag = expectedEtag;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
if (request.RequestUri == _indexUri) {
{ if (request.RequestUri == _indexUri)
return Task.FromResult(CreateJsonResponse(_indexPayload)); {
} return Task.FromResult(CreateJsonResponse(_indexPayload));
}
if (request.RequestUri == _catalogUri)
{ if (request.RequestUri == _catalogUri)
return Task.FromResult(CreateJsonResponse(_catalogPayload)); {
} return Task.FromResult(CreateJsonResponse(_catalogPayload));
}
if (request.RequestUri == _documentUri)
{ if (request.RequestUri == _documentUri)
DocumentRequestCount++; {
if (request.Headers.IfNoneMatch is { Count: > 0 }) DocumentRequestCount++;
{ if (request.Headers.IfNoneMatch is { Count: > 0 })
var header = request.Headers.IfNoneMatch.First().ToString(); {
SeenIfNoneMatch.Add(header); var header = request.Headers.IfNoneMatch.First().ToString();
if (header.Trim('"') == _expectedEtag || header == $"\"{_expectedEtag}\"") SeenIfNoneMatch.Add(header);
{ if (header.Trim('"') == _expectedEtag || header == $"\"{_expectedEtag}\"")
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified)); {
} return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified));
} }
}
var response = new HttpResponseMessage(HttpStatusCode.OK)
{ var response = new HttpResponseMessage(HttpStatusCode.OK)
Content = new ByteArrayContent(_documentPayload), {
}; Content = new ByteArrayContent(_documentPayload),
response.Headers.ETag = new EntityTagHeaderValue($"\"{_expectedEtag}\""); };
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); response.Headers.ETag = new EntityTagHeaderValue($"\"{_expectedEtag}\"");
return Task.FromResult(response); response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
} return Task.FromResult(response);
}
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
{ return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
Content = new StringContent($"No response configured for {request.RequestUri}"), {
}); Content = new StringContent($"No response configured for {request.RequestUri}"),
} });
}
private static HttpResponseMessage CreateJsonResponse(string payload)
=> new(HttpStatusCode.OK) private static HttpResponseMessage CreateJsonResponse(string payload)
{ => new(HttpStatusCode.OK)
Content = new StringContent(payload, Encoding.UTF8, "application/json"), {
}; Content = new StringContent(payload, Encoding.UTF8, "application/json"),
} };
}
private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
{ private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
public VexConnectorState? CurrentState { get; private set; } {
public VexConnectorState? CurrentState { get; private set; }
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
=> ValueTask.FromResult(CurrentState); public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(CurrentState);
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
{ public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
CurrentState = state; {
return ValueTask.CompletedTask; CurrentState = state;
} return ValueTask.CompletedTask;
} }
}
private sealed class InMemoryRawSink : IVexRawDocumentSink
{ private sealed class InMemoryRawSink : IVexRawDocumentSink
public List<VexRawDocument> Documents { get; } = new(); {
public List<VexRawDocument> Documents { get; } = new();
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
{ public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
Documents.Add(document); {
return ValueTask.CompletedTask; Documents.Add(document);
} return ValueTask.CompletedTask;
} }
}
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
{ private sealed class NoopSignatureVerifier : IVexSignatureVerifier
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) {
=> ValueTask.FromResult<VexSignatureMetadata?>(null); public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
} => ValueTask.FromResult<VexSignatureMetadata?>(null);
}
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
{ private sealed class NoopNormalizerRouter : IVexNormalizerRouter
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) {
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty)); public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
} => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
} }
}

View File

@@ -1,277 +1,277 @@
using System; using System;
using System.Collections.Immutable; using System.Collections.Immutable;
using System.IO; using System.IO;
using System.Text; using System.Text;
using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Driver; using MongoDB.Driver;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export; using StellaOps.Excititor.Export;
using StellaOps.Excititor.Policy; using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Storage.Mongo;
using Xunit; using Xunit;
namespace StellaOps.Excititor.Export.Tests; namespace StellaOps.Excititor.Export.Tests;
public sealed class ExportEngineTests public sealed class ExportEngineTests
{ {
[Fact] [Fact]
public async Task ExportAsync_GeneratesAndCachesManifest() public async Task ExportAsync_GeneratesAndCachesManifest()
{ {
var store = new InMemoryExportStore(); var store = new InMemoryExportStore();
var evaluator = new StaticPolicyEvaluator("baseline/v1"); var evaluator = new StaticPolicyEvaluator("baseline/v1");
var dataSource = new InMemoryExportDataSource(); var dataSource = new InMemoryExportDataSource();
var exporter = new DummyExporter(VexExportFormat.Json); var exporter = new DummyExporter(VexExportFormat.Json);
var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger<VexExportEngine>.Instance); var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger<VexExportEngine>.Instance);
var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") });
var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow, ForceRefresh: false); var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow, ForceRefresh: false);
var manifest = await engine.ExportAsync(context, CancellationToken.None); var manifest = await engine.ExportAsync(context, CancellationToken.None);
Assert.False(manifest.FromCache); Assert.False(manifest.FromCache);
Assert.Equal(VexExportFormat.Json, manifest.Format); Assert.Equal(VexExportFormat.Json, manifest.Format);
Assert.Equal("baseline/v1", manifest.ConsensusRevision); Assert.Equal("baseline/v1", manifest.ConsensusRevision);
Assert.Equal(1, manifest.ClaimCount); Assert.Equal(1, manifest.ClaimCount);
// second call hits cache // second call hits cache
var cached = await engine.ExportAsync(context, CancellationToken.None); var cached = await engine.ExportAsync(context, CancellationToken.None);
Assert.True(cached.FromCache); Assert.True(cached.FromCache);
Assert.Equal(manifest.ExportId, cached.ExportId); Assert.Equal(manifest.ExportId, cached.ExportId);
} }
[Fact] [Fact]
public async Task ExportAsync_ForceRefreshInvalidatesCacheEntry() public async Task ExportAsync_ForceRefreshInvalidatesCacheEntry()
{ {
var store = new InMemoryExportStore(); var store = new InMemoryExportStore();
var evaluator = new StaticPolicyEvaluator("baseline/v1"); var evaluator = new StaticPolicyEvaluator("baseline/v1");
var dataSource = new InMemoryExportDataSource(); var dataSource = new InMemoryExportDataSource();
var exporter = new DummyExporter(VexExportFormat.Json); var exporter = new DummyExporter(VexExportFormat.Json);
var cacheIndex = new RecordingCacheIndex(); var cacheIndex = new RecordingCacheIndex();
var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger<VexExportEngine>.Instance, cacheIndex); var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger<VexExportEngine>.Instance, cacheIndex);
var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") });
var initialContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); var initialContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow);
_ = await engine.ExportAsync(initialContext, CancellationToken.None); _ = await engine.ExportAsync(initialContext, CancellationToken.None);
var refreshContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow.AddMinutes(1), ForceRefresh: true); var refreshContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow.AddMinutes(1), ForceRefresh: true);
var refreshed = await engine.ExportAsync(refreshContext, CancellationToken.None); var refreshed = await engine.ExportAsync(refreshContext, CancellationToken.None);
Assert.False(refreshed.FromCache); Assert.False(refreshed.FromCache);
var signature = VexQuerySignature.FromQuery(refreshContext.Query); var signature = VexQuerySignature.FromQuery(refreshContext.Query);
Assert.True(cacheIndex.RemoveCalls.TryGetValue((signature.Value, refreshContext.Format), out var removed)); Assert.True(cacheIndex.RemoveCalls.TryGetValue((signature.Value, refreshContext.Format), out var removed));
Assert.True(removed); Assert.True(removed);
} }
[Fact] [Fact]
public async Task ExportAsync_WritesArtifactsToAllStores() public async Task ExportAsync_WritesArtifactsToAllStores()
{ {
var store = new InMemoryExportStore(); var store = new InMemoryExportStore();
var evaluator = new StaticPolicyEvaluator("baseline/v1"); var evaluator = new StaticPolicyEvaluator("baseline/v1");
var dataSource = new InMemoryExportDataSource(); var dataSource = new InMemoryExportDataSource();
var exporter = new DummyExporter(VexExportFormat.Json); var exporter = new DummyExporter(VexExportFormat.Json);
var recorder1 = new RecordingArtifactStore(); var recorder1 = new RecordingArtifactStore();
var recorder2 = new RecordingArtifactStore(); var recorder2 = new RecordingArtifactStore();
var engine = new VexExportEngine( var engine = new VexExportEngine(
store, store,
evaluator, evaluator,
dataSource, dataSource,
new[] { exporter }, new[] { exporter },
NullLogger<VexExportEngine>.Instance, NullLogger<VexExportEngine>.Instance,
cacheIndex: null, cacheIndex: null,
artifactStores: new[] { recorder1, recorder2 }); artifactStores: new[] { recorder1, recorder2 });
var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") });
var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow);
await engine.ExportAsync(context, CancellationToken.None); await engine.ExportAsync(context, CancellationToken.None);
Assert.Equal(1, recorder1.SaveCount); Assert.Equal(1, recorder1.SaveCount);
Assert.Equal(1, recorder2.SaveCount); Assert.Equal(1, recorder2.SaveCount);
} }
[Fact] [Fact]
public async Task ExportAsync_AttachesAttestationMetadata() public async Task ExportAsync_AttachesAttestationMetadata()
{ {
var store = new InMemoryExportStore(); var store = new InMemoryExportStore();
var evaluator = new StaticPolicyEvaluator("baseline/v1"); var evaluator = new StaticPolicyEvaluator("baseline/v1");
var dataSource = new InMemoryExportDataSource(); var dataSource = new InMemoryExportDataSource();
var exporter = new DummyExporter(VexExportFormat.Json); var exporter = new DummyExporter(VexExportFormat.Json);
var attestation = new RecordingAttestationClient(); var attestation = new RecordingAttestationClient();
var engine = new VexExportEngine( var engine = new VexExportEngine(
store, store,
evaluator, evaluator,
dataSource, dataSource,
new[] { exporter }, new[] { exporter },
NullLogger<VexExportEngine>.Instance, NullLogger<VexExportEngine>.Instance,
cacheIndex: null, cacheIndex: null,
artifactStores: null, artifactStores: null,
attestationClient: attestation); attestationClient: attestation);
var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") });
var requestedAt = DateTimeOffset.UtcNow; var requestedAt = DateTimeOffset.UtcNow;
var context = new VexExportRequestContext(query, VexExportFormat.Json, requestedAt); var context = new VexExportRequestContext(query, VexExportFormat.Json, requestedAt);
var manifest = await engine.ExportAsync(context, CancellationToken.None); var manifest = await engine.ExportAsync(context, CancellationToken.None);
Assert.NotNull(attestation.LastRequest); Assert.NotNull(attestation.LastRequest);
Assert.Equal(manifest.ExportId, attestation.LastRequest!.ExportId); Assert.Equal(manifest.ExportId, attestation.LastRequest!.ExportId);
Assert.NotNull(manifest.Attestation); Assert.NotNull(manifest.Attestation);
Assert.Equal(attestation.Response.Attestation.EnvelopeDigest, manifest.Attestation!.EnvelopeDigest); Assert.Equal(attestation.Response.Attestation.EnvelopeDigest, manifest.Attestation!.EnvelopeDigest);
Assert.Equal(attestation.Response.Attestation.PredicateType, manifest.Attestation.PredicateType); Assert.Equal(attestation.Response.Attestation.PredicateType, manifest.Attestation.PredicateType);
Assert.NotNull(store.LastSavedManifest); Assert.NotNull(store.LastSavedManifest);
Assert.Equal(manifest.Attestation, store.LastSavedManifest!.Attestation); Assert.Equal(manifest.Attestation, store.LastSavedManifest!.Attestation);
} }
private sealed class InMemoryExportStore : IVexExportStore private sealed class InMemoryExportStore : IVexExportStore
{ {
private readonly Dictionary<string, VexExportManifest> _store = new(StringComparer.Ordinal); private readonly Dictionary<string, VexExportManifest> _store = new(StringComparer.Ordinal);
public VexExportManifest? LastSavedManifest { get; private set; } public VexExportManifest? LastSavedManifest { get; private set; }
public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
var key = CreateKey(signature.Value, format); var key = CreateKey(signature.Value, format);
_store.TryGetValue(key, out var manifest); _store.TryGetValue(key, out var manifest);
return ValueTask.FromResult<VexExportManifest?>(manifest); return ValueTask.FromResult<VexExportManifest?>(manifest);
} }
public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
var key = CreateKey(manifest.QuerySignature.Value, manifest.Format); var key = CreateKey(manifest.QuerySignature.Value, manifest.Format);
_store[key] = manifest; _store[key] = manifest;
LastSavedManifest = manifest; LastSavedManifest = manifest;
return ValueTask.CompletedTask; return ValueTask.CompletedTask;
} }
private static string CreateKey(string signature, VexExportFormat format) private static string CreateKey(string signature, VexExportFormat format)
=> FormattableString.Invariant($"{signature}|{format}"); => FormattableString.Invariant($"{signature}|{format}");
} }
private sealed class RecordingAttestationClient : IVexAttestationClient private sealed class RecordingAttestationClient : IVexAttestationClient
{ {
public VexAttestationRequest? LastRequest { get; private set; } public VexAttestationRequest? LastRequest { get; private set; }
public VexAttestationResponse Response { get; } = new VexAttestationResponse( public VexAttestationResponse Response { get; } = new VexAttestationResponse(
new VexAttestationMetadata( new VexAttestationMetadata(
predicateType: "https://stella-ops.org/attestations/vex-export", predicateType: "https://stella-ops.org/attestations/vex-export",
rekor: new VexRekorReference("0.2", "rekor://entry", "123"), rekor: new VexRekorReference("0.2", "rekor://entry", "123"),
envelopeDigest: "sha256:envelope", envelopeDigest: "sha256:envelope",
signedAt: DateTimeOffset.UnixEpoch), signedAt: DateTimeOffset.UnixEpoch),
ImmutableDictionary<string, string>.Empty); ImmutableDictionary<string, string>.Empty);
public ValueTask<VexAttestationResponse> SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) public ValueTask<VexAttestationResponse> SignAsync(VexAttestationRequest request, CancellationToken cancellationToken)
{ {
LastRequest = request; LastRequest = request;
return ValueTask.FromResult(Response); return ValueTask.FromResult(Response);
} }
public ValueTask<VexAttestationVerification> VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) public ValueTask<VexAttestationVerification> VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken)
=> ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary<string, string>.Empty)); => ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary<string, string>.Empty));
} }
private sealed class RecordingCacheIndex : IVexCacheIndex private sealed class RecordingCacheIndex : IVexCacheIndex
{ {
public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new(); public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new();
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<VexCacheEntry?>(null); => ValueTask.FromResult<VexCacheEntry?>(null);
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.CompletedTask; => ValueTask.CompletedTask;
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
RemoveCalls[(signature.Value, format)] = true; RemoveCalls[(signature.Value, format)] = true;
return ValueTask.CompletedTask; return ValueTask.CompletedTask;
} }
} }
private sealed class RecordingArtifactStore : IVexArtifactStore private sealed class RecordingArtifactStore : IVexArtifactStore
{ {
public int SaveCount { get; private set; } public int SaveCount { get; private set; }
public ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) public ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken)
{ {
SaveCount++; SaveCount++;
return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory", artifact.Content.Length, artifact.Metadata)); return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory", artifact.Content.Length, artifact.Metadata));
} }
public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
=> ValueTask.CompletedTask; => ValueTask.CompletedTask;
public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
=> ValueTask.FromResult<Stream?>(null); => ValueTask.FromResult<Stream?>(null);
} }
private sealed class StaticPolicyEvaluator : IVexPolicyEvaluator private sealed class StaticPolicyEvaluator : IVexPolicyEvaluator
{ {
public StaticPolicyEvaluator(string version) public StaticPolicyEvaluator(string version)
{ {
Version = version; Version = version;
} }
public string Version { get; } public string Version { get; }
public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default;
public double GetProviderWeight(VexProvider provider) => 1.0; public double GetProviderWeight(VexProvider provider) => 1.0;
public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason)
{ {
rejectionReason = null; rejectionReason = null;
return true; return true;
} }
} }
private sealed class InMemoryExportDataSource : IVexExportDataSource private sealed class InMemoryExportDataSource : IVexExportDataSource
{ {
public ValueTask<VexExportDataSet> FetchAsync(VexQuery query, CancellationToken cancellationToken) public ValueTask<VexExportDataSet> FetchAsync(VexQuery query, CancellationToken cancellationToken)
{ {
var claim = new VexClaim( var claim = new VexClaim(
"CVE-2025-0001", "CVE-2025-0001",
"vendor", "vendor",
new VexProduct("pkg:demo/app", "Demo"), new VexProduct("pkg:demo/app", "Demo"),
VexClaimStatus.Affected, VexClaimStatus.Affected,
new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:demo", new Uri("https://example.org/demo")), new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:demo", new Uri("https://example.org/demo")),
DateTimeOffset.UtcNow, DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow); DateTimeOffset.UtcNow);
var consensus = new VexConsensus( var consensus = new VexConsensus(
"CVE-2025-0001", "CVE-2025-0001",
claim.Product, claim.Product,
VexConsensusStatus.Affected, VexConsensusStatus.Affected,
DateTimeOffset.UtcNow, DateTimeOffset.UtcNow,
new[] { new VexConsensusSource("vendor", VexClaimStatus.Affected, "sha256:demo", 1.0) }, new[] { new VexConsensusSource("vendor", VexClaimStatus.Affected, "sha256:demo", 1.0) },
conflicts: null, conflicts: null,
policyVersion: "baseline/v1", policyVersion: "baseline/v1",
summary: "affected"); summary: "affected");
return ValueTask.FromResult(new VexExportDataSet( return ValueTask.FromResult(new VexExportDataSet(
ImmutableArray.Create(consensus), ImmutableArray.Create(consensus),
ImmutableArray.Create(claim), ImmutableArray.Create(claim),
ImmutableArray.Create("vendor"))); ImmutableArray.Create("vendor")));
} }
} }
private sealed class DummyExporter : IVexExporter private sealed class DummyExporter : IVexExporter
{ {
public DummyExporter(VexExportFormat format) public DummyExporter(VexExportFormat format)
{ {
Format = format; Format = format;
} }
public VexExportFormat Format { get; } public VexExportFormat Format { get; }
public VexContentAddress Digest(VexExportRequest request) public VexContentAddress Digest(VexExportRequest request)
=> new("sha256", "deadbeef"); => new("sha256", "deadbeef");
public ValueTask<VexExportResult> SerializeAsync(VexExportRequest request, Stream output, CancellationToken cancellationToken) public ValueTask<VexExportResult> SerializeAsync(VexExportRequest request, Stream output, CancellationToken cancellationToken)
{ {
var bytes = System.Text.Encoding.UTF8.GetBytes("{}"); var bytes = System.Text.Encoding.UTF8.GetBytes("{}");
output.Write(bytes); output.Write(bytes);
return ValueTask.FromResult(new VexExportResult(new VexContentAddress("sha256", "deadbeef"), bytes.Length, ImmutableDictionary<string, string>.Empty)); return ValueTask.FromResult(new VexExportResult(new VexContentAddress("sha256", "deadbeef"), bytes.Length, ImmutableDictionary<string, string>.Empty));
} }
} }
} }

View File

@@ -1,82 +1,82 @@
using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Driver; using MongoDB.Driver;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export; using StellaOps.Excititor.Export;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Export.Tests; namespace StellaOps.Excititor.Export.Tests;
public sealed class VexExportCacheServiceTests public sealed class VexExportCacheServiceTests
{ {
[Fact] [Fact]
public async Task InvalidateAsync_RemovesEntry() public async Task InvalidateAsync_RemovesEntry()
{ {
var cacheIndex = new RecordingIndex(); var cacheIndex = new RecordingIndex();
var maintenance = new StubMaintenance(); var maintenance = new StubMaintenance();
var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance); var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance);
var signature = new VexQuerySignature("format=json|provider=vendor"); var signature = new VexQuerySignature("format=json|provider=vendor");
await service.InvalidateAsync(signature, VexExportFormat.Json, CancellationToken.None); await service.InvalidateAsync(signature, VexExportFormat.Json, CancellationToken.None);
Assert.Equal(signature.Value, cacheIndex.LastSignature?.Value); Assert.Equal(signature.Value, cacheIndex.LastSignature?.Value);
Assert.Equal(VexExportFormat.Json, cacheIndex.LastFormat); Assert.Equal(VexExportFormat.Json, cacheIndex.LastFormat);
Assert.Equal(1, cacheIndex.RemoveCalls); Assert.Equal(1, cacheIndex.RemoveCalls);
} }
[Fact] [Fact]
public async Task PruneExpiredAsync_ReturnsCount() public async Task PruneExpiredAsync_ReturnsCount()
{ {
var cacheIndex = new RecordingIndex(); var cacheIndex = new RecordingIndex();
var maintenance = new StubMaintenance { ExpiredCount = 3 }; var maintenance = new StubMaintenance { ExpiredCount = 3 };
var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance); var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance);
var removed = await service.PruneExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None); var removed = await service.PruneExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None);
Assert.Equal(3, removed); Assert.Equal(3, removed);
} }
[Fact] [Fact]
public async Task PruneDanglingAsync_ReturnsCount() public async Task PruneDanglingAsync_ReturnsCount()
{ {
var cacheIndex = new RecordingIndex(); var cacheIndex = new RecordingIndex();
var maintenance = new StubMaintenance { DanglingCount = 2 }; var maintenance = new StubMaintenance { DanglingCount = 2 };
var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance); var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance);
var removed = await service.PruneDanglingAsync(CancellationToken.None); var removed = await service.PruneDanglingAsync(CancellationToken.None);
Assert.Equal(2, removed); Assert.Equal(2, removed);
} }
private sealed class RecordingIndex : IVexCacheIndex private sealed class RecordingIndex : IVexCacheIndex
{ {
public VexQuerySignature? LastSignature { get; private set; } public VexQuerySignature? LastSignature { get; private set; }
public VexExportFormat LastFormat { get; private set; } public VexExportFormat LastFormat { get; private set; }
public int RemoveCalls { get; private set; } public int RemoveCalls { get; private set; }
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<VexCacheEntry?>(null); => ValueTask.FromResult<VexCacheEntry?>(null);
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.CompletedTask; => ValueTask.CompletedTask;
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{ {
LastSignature = signature; LastSignature = signature;
LastFormat = format; LastFormat = format;
RemoveCalls++; RemoveCalls++;
return ValueTask.CompletedTask; return ValueTask.CompletedTask;
} }
} }
private sealed class StubMaintenance : IVexCacheMaintenance private sealed class StubMaintenance : IVexCacheMaintenance
{ {
public int ExpiredCount { get; set; } public int ExpiredCount { get; set; }
public int DanglingCount { get; set; } public int DanglingCount { get; set; }
public ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) public ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(ExpiredCount); => ValueTask.FromResult(ExpiredCount);
public ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) public ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(DanglingCount); => ValueTask.FromResult(DanglingCount);
} }
} }

View File

@@ -1,209 +1,244 @@
using System.Collections.Immutable; using System;
using System.IO; using System.Collections.Immutable;
using System.Linq; using System.Collections.Generic;
using Microsoft.Extensions.DependencyInjection; using System.IO;
using Microsoft.Extensions.Logging; using System.Linq;
using StellaOps.Excititor.Core; using Microsoft.Extensions.DependencyInjection;
using StellaOps.Excititor.Policy; using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Policy;
namespace StellaOps.Excititor.Export; using StellaOps.Excititor.Storage.Mongo;
public interface IExportEngine namespace StellaOps.Excititor.Export;
{
ValueTask<VexExportManifest> ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken); public interface IExportEngine
} {
ValueTask<VexExportManifest> ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken);
public sealed record VexExportRequestContext( }
VexQuery Query,
VexExportFormat Format, public sealed record VexExportRequestContext(
DateTimeOffset RequestedAt, VexQuery Query,
bool ForceRefresh = false); VexExportFormat Format,
DateTimeOffset RequestedAt,
public interface IVexExportDataSource bool ForceRefresh = false);
{
ValueTask<VexExportDataSet> FetchAsync(VexQuery query, CancellationToken cancellationToken); public interface IVexExportDataSource
} {
ValueTask<VexExportDataSet> FetchAsync(VexQuery query, CancellationToken cancellationToken);
public sealed record VexExportDataSet( }
ImmutableArray<VexConsensus> Consensus,
ImmutableArray<VexClaim> Claims, public sealed record VexExportDataSet(
ImmutableArray<string> SourceProviders); ImmutableArray<VexConsensus> Consensus,
ImmutableArray<VexClaim> Claims,
public sealed class VexExportEngine : IExportEngine ImmutableArray<string> SourceProviders);
{
private readonly IVexExportStore _exportStore; public sealed class VexExportEngine : IExportEngine
private readonly IVexPolicyEvaluator _policyEvaluator; {
private readonly IVexExportDataSource _dataSource; private readonly IVexExportStore _exportStore;
private readonly IReadOnlyDictionary<VexExportFormat, IVexExporter> _exporters; private readonly IVexPolicyEvaluator _policyEvaluator;
private readonly ILogger<VexExportEngine> _logger; private readonly IVexExportDataSource _dataSource;
private readonly IVexCacheIndex? _cacheIndex; private readonly IReadOnlyDictionary<VexExportFormat, IVexExporter> _exporters;
private readonly IReadOnlyList<IVexArtifactStore> _artifactStores; private readonly ILogger<VexExportEngine> _logger;
private readonly IVexAttestationClient? _attestationClient; private readonly IVexCacheIndex? _cacheIndex;
private readonly IReadOnlyList<IVexArtifactStore> _artifactStores;
public VexExportEngine( private readonly IVexAttestationClient? _attestationClient;
IVexExportStore exportStore,
IVexPolicyEvaluator policyEvaluator, public VexExportEngine(
IVexExportDataSource dataSource, IVexExportStore exportStore,
IEnumerable<IVexExporter> exporters, IVexPolicyEvaluator policyEvaluator,
ILogger<VexExportEngine> logger, IVexExportDataSource dataSource,
IVexCacheIndex? cacheIndex = null, IEnumerable<IVexExporter> exporters,
IEnumerable<IVexArtifactStore>? artifactStores = null, ILogger<VexExportEngine> logger,
IVexAttestationClient? attestationClient = null) IVexCacheIndex? cacheIndex = null,
{ IEnumerable<IVexArtifactStore>? artifactStores = null,
_exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore)); IVexAttestationClient? attestationClient = null)
_policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator)); {
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); _exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger)); _policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator));
_cacheIndex = cacheIndex; _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
_artifactStores = artifactStores?.ToArray() ?? Array.Empty<IVexArtifactStore>(); _logger = logger ?? throw new ArgumentNullException(nameof(logger));
_attestationClient = attestationClient; _cacheIndex = cacheIndex;
_artifactStores = artifactStores?.ToArray() ?? Array.Empty<IVexArtifactStore>();
if (exporters is null) _attestationClient = attestationClient;
{
throw new ArgumentNullException(nameof(exporters)); if (exporters is null)
} {
throw new ArgumentNullException(nameof(exporters));
_exporters = exporters.ToDictionary(x => x.Format); }
}
_exporters = exporters.ToDictionary(x => x.Format);
public async ValueTask<VexExportManifest> ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken) }
{
ArgumentNullException.ThrowIfNull(context); public async ValueTask<VexExportManifest> ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken)
var signature = VexQuerySignature.FromQuery(context.Query); {
ArgumentNullException.ThrowIfNull(context);
if (!context.ForceRefresh) var signature = VexQuerySignature.FromQuery(context.Query);
{
var cached = await _exportStore.FindAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); if (!context.ForceRefresh)
if (cached is not null) {
{ var cached = await _exportStore.FindAsync(signature, context.Format, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Reusing cached export for {Signature} ({Format})", signature.Value, context.Format); if (cached is not null)
return new VexExportManifest( {
cached.ExportId, _logger.LogInformation("Reusing cached export for {Signature} ({Format})", signature.Value, context.Format);
cached.QuerySignature, return new VexExportManifest(
cached.Format, cached.ExportId,
cached.CreatedAt, cached.QuerySignature,
cached.Artifact, cached.Format,
cached.ClaimCount, cached.CreatedAt,
cached.SourceProviders, cached.Artifact,
fromCache: true, cached.ClaimCount,
cached.ConsensusRevision, cached.SourceProviders,
cached.Attestation, fromCache: true,
cached.SizeBytes); cached.ConsensusRevision,
} cached.PolicyRevisionId,
} cached.PolicyDigest,
else if (_cacheIndex is not null) cached.ConsensusDigest,
{ cached.ScoreDigest,
await _cacheIndex.RemoveAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); cached.Attestation,
_logger.LogInformation("Force refresh requested; invalidated cache entry for {Signature} ({Format})", signature.Value, context.Format); cached.SizeBytes);
} }
}
var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false); else if (_cacheIndex is not null)
var exporter = ResolveExporter(context.Format); {
await _cacheIndex.RemoveAsync(signature, context.Format, cancellationToken).ConfigureAwait(false);
var exportRequest = new VexExportRequest( _logger.LogInformation("Force refresh requested; invalidated cache entry for {Signature} ({Format})", signature.Value, context.Format);
context.Query, }
dataset.Consensus,
dataset.Claims, var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false);
context.RequestedAt); var exporter = ResolveExporter(context.Format);
var policySnapshot = _policyEvaluator.Snapshot;
var digest = exporter.Digest(exportRequest);
var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}"); var exportRequest = new VexExportRequest(
context.Query,
await using var buffer = new MemoryStream(); dataset.Consensus,
var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false); dataset.Claims,
context.RequestedAt);
if (_artifactStores.Count > 0)
{ var digest = exporter.Digest(exportRequest);
var writtenBytes = buffer.ToArray(); var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}");
try
{ await using var buffer = new MemoryStream();
var artifact = new VexExportArtifact( var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false);
result.Digest,
context.Format, if (_artifactStores.Count > 0)
writtenBytes, {
result.Metadata); var writtenBytes = buffer.ToArray();
try
foreach (var store in _artifactStores) {
{ var artifact = new VexExportArtifact(
await store.SaveAsync(artifact, cancellationToken).ConfigureAwait(false); result.Digest,
} context.Format,
writtenBytes,
_logger.LogInformation("Stored export artifact {Digest} via {StoreCount} store(s)", result.Digest.ToUri(), _artifactStores.Count); result.Metadata);
}
catch (Exception ex) foreach (var store in _artifactStores)
{ {
_logger.LogError(ex, "Failed to store export artifact {Digest}", result.Digest.ToUri()); await store.SaveAsync(artifact, cancellationToken).ConfigureAwait(false);
throw; }
}
} _logger.LogInformation("Stored export artifact {Digest} via {StoreCount} store(s)", result.Digest.ToUri(), _artifactStores.Count);
}
VexAttestationMetadata? attestationMetadata = null; catch (Exception ex)
if (_attestationClient is not null) {
{ _logger.LogError(ex, "Failed to store export artifact {Digest}", result.Digest.ToUri());
var attestationRequest = new VexAttestationRequest( throw;
exportId, }
signature, }
digest,
context.Format, VexAttestationMetadata? attestationMetadata = null;
context.RequestedAt, if (_attestationClient is not null)
dataset.SourceProviders, {
result.Metadata); var attestationRequest = new VexAttestationRequest(
exportId,
var response = await _attestationClient.SignAsync(attestationRequest, cancellationToken).ConfigureAwait(false); signature,
attestationMetadata = response.Attestation; digest,
context.Format,
if (!response.Diagnostics.IsEmpty) context.RequestedAt,
{ dataset.SourceProviders,
foreach (var diagnostic in response.Diagnostics) result.Metadata);
{
_logger.LogDebug( var response = await _attestationClient.SignAsync(attestationRequest, cancellationToken).ConfigureAwait(false);
"Attestation diagnostic {Key}={Value} for export {ExportId}", attestationMetadata = response.Attestation;
diagnostic.Key,
diagnostic.Value, if (!response.Diagnostics.IsEmpty)
exportId); {
} foreach (var diagnostic in response.Diagnostics)
} {
_logger.LogDebug(
_logger.LogInformation("Attestation generated for export {ExportId}", exportId); "Attestation diagnostic {Key}={Value} for export {ExportId}",
} diagnostic.Key,
diagnostic.Value,
var manifest = new VexExportManifest( exportId);
exportId, }
signature, }
context.Format,
context.RequestedAt, _logger.LogInformation("Attestation generated for export {ExportId}", exportId);
digest, }
dataset.Claims.Length,
dataset.SourceProviders, var consensusDigestAddress = TryGetContentAddress(result.Metadata, "consensusDigest");
fromCache: false, var scoreDigestAddress = TryGetContentAddress(result.Metadata, "scoreDigest");
consensusRevision: _policyEvaluator.Version,
attestation: attestationMetadata, var manifest = new VexExportManifest(
sizeBytes: result.BytesWritten); exportId,
signature,
await _exportStore.SaveAsync(manifest, cancellationToken).ConfigureAwait(false); context.Format,
context.RequestedAt,
_logger.LogInformation( digest,
"Export generated for {Signature} ({Format}) size={SizeBytes} bytes", dataset.Claims.Length,
signature.Value, dataset.SourceProviders,
context.Format, fromCache: false,
result.BytesWritten); consensusRevision: policySnapshot.Version,
policyRevisionId: policySnapshot.RevisionId,
return manifest; policyDigest: policySnapshot.Digest,
} consensusDigest: consensusDigestAddress,
scoreDigest: scoreDigestAddress,
private IVexExporter ResolveExporter(VexExportFormat format) attestation: attestationMetadata,
=> _exporters.TryGetValue(format, out var exporter) sizeBytes: result.BytesWritten);
? exporter
: throw new InvalidOperationException($"No exporter registered for format '{format}'."); await _exportStore.SaveAsync(manifest, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation(
public static class VexExportServiceCollectionExtensions "Export generated for {Signature} ({Format}) size={SizeBytes} bytes",
{ signature.Value,
public static IServiceCollection AddVexExportEngine(this IServiceCollection services) context.Format,
{ result.BytesWritten);
services.AddSingleton<IExportEngine, VexExportEngine>();
services.AddVexExportCacheServices(); return manifest;
return services; }
}
} private static VexContentAddress? TryGetContentAddress(IReadOnlyDictionary<string, string> metadata, string key)
{
if (metadata is null)
{
return null;
}
if (!metadata.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value))
{
return null;
}
var parts = value.Split(':', 2, StringSplitOptions.TrimEntries);
if (parts.Length != 2)
{
return null;
}
return new VexContentAddress(parts[0], parts[1]);
}
private IVexExporter ResolveExporter(VexExportFormat format)
=> _exporters.TryGetValue(format, out var exporter)
? exporter
: throw new InvalidOperationException($"No exporter registered for format '{format}'.");
}
public static class VexExportServiceCollectionExtensions
{
public static IServiceCollection AddVexExportEngine(this IServiceCollection services)
{
services.AddSingleton<IExportEngine, VexExportEngine>();
services.AddVexExportCacheServices();
return services;
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,116 +1,119 @@
using System; using System;
using System.Collections.Immutable; using System.Collections.Immutable;
using System.Linq; using System.Linq;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using StellaOps.Plugin; using StellaOps.Plugin;
using StellaOps.Excititor.Core; using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Worker.Scheduling;
namespace StellaOps.Excititor.Worker.Scheduling;
internal sealed class DefaultVexProviderRunner : IVexProviderRunner
{ internal sealed class DefaultVexProviderRunner : IVexProviderRunner
private readonly IServiceProvider _serviceProvider; {
private readonly PluginCatalog _pluginCatalog; private readonly IServiceProvider _serviceProvider;
private readonly ILogger<DefaultVexProviderRunner> _logger; private readonly PluginCatalog _pluginCatalog;
private readonly TimeProvider _timeProvider; private readonly ILogger<DefaultVexProviderRunner> _logger;
private readonly TimeProvider _timeProvider;
public DefaultVexProviderRunner(
IServiceProvider serviceProvider, public DefaultVexProviderRunner(
PluginCatalog pluginCatalog, IServiceProvider serviceProvider,
ILogger<DefaultVexProviderRunner> logger, PluginCatalog pluginCatalog,
TimeProvider timeProvider) ILogger<DefaultVexProviderRunner> logger,
{ TimeProvider timeProvider)
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); {
_pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog)); _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger)); _pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); _logger = logger ?? throw new ArgumentNullException(nameof(logger));
} _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public async ValueTask RunAsync(string providerId, CancellationToken cancellationToken)
{ public async ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken)
ArgumentException.ThrowIfNullOrWhiteSpace(providerId); {
ArgumentNullException.ThrowIfNull(schedule);
using var scope = _serviceProvider.CreateScope(); ArgumentException.ThrowIfNullOrWhiteSpace(schedule.ProviderId);
var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider);
var matched = availablePlugins.FirstOrDefault(plugin => using var scope = _serviceProvider.CreateScope();
string.Equals(plugin.Name, providerId, StringComparison.OrdinalIgnoreCase)); var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider);
var matched = availablePlugins.FirstOrDefault(plugin =>
if (matched is not null) string.Equals(plugin.Name, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
{
_logger.LogInformation( if (matched is not null)
"Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.", {
matched.Name, _logger.LogInformation(
providerId); "Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.",
} matched.Name,
else schedule.ProviderId);
{ }
_logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", providerId); else
} {
_logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", schedule.ProviderId);
var connectors = scope.ServiceProvider.GetServices<IVexConnector>(); }
var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, providerId, StringComparison.OrdinalIgnoreCase));
var connectors = scope.ServiceProvider.GetServices<IVexConnector>();
if (connector is null) var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, schedule.ProviderId, StringComparison.OrdinalIgnoreCase));
{
_logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", providerId); if (connector is null)
return; {
} _logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", schedule.ProviderId);
return;
await ExecuteConnectorAsync(scope.ServiceProvider, connector, cancellationToken).ConfigureAwait(false); }
}
await ExecuteConnectorAsync(scope.ServiceProvider, connector, schedule.Settings, cancellationToken).ConfigureAwait(false);
private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, CancellationToken cancellationToken) }
{
var rawStore = scopeProvider.GetRequiredService<IVexRawStore>(); private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, VexConnectorSettings settings, CancellationToken cancellationToken)
var claimStore = scopeProvider.GetRequiredService<IVexClaimStore>(); {
var providerStore = scopeProvider.GetRequiredService<IVexProviderStore>(); var effectiveSettings = settings ?? VexConnectorSettings.Empty;
var normalizerRouter = scopeProvider.GetRequiredService<IVexNormalizerRouter>(); var rawStore = scopeProvider.GetRequiredService<IVexRawStore>();
var signatureVerifier = scopeProvider.GetRequiredService<IVexSignatureVerifier>(); var claimStore = scopeProvider.GetRequiredService<IVexClaimStore>();
var sessionProvider = scopeProvider.GetRequiredService<IVexMongoSessionProvider>(); var providerStore = scopeProvider.GetRequiredService<IVexProviderStore>();
var session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); var normalizerRouter = scopeProvider.GetRequiredService<IVexNormalizerRouter>();
var signatureVerifier = scopeProvider.GetRequiredService<IVexSignatureVerifier>();
var descriptor = connector switch var sessionProvider = scopeProvider.GetRequiredService<IVexMongoSessionProvider>();
{ var session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
VexConnectorBase baseConnector => baseConnector.Descriptor,
_ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id) var descriptor = connector switch
}; {
VexConnectorBase baseConnector => baseConnector.Descriptor,
var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false) _ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id)
?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind); };
await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false); var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false)
?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
await connector.ValidateAsync(VexConnectorSettings.Empty, cancellationToken).ConfigureAwait(false);
await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
var context = new VexConnectorContext(
Since: null, await connector.ValidateAsync(effectiveSettings, cancellationToken).ConfigureAwait(false);
Settings: VexConnectorSettings.Empty,
RawSink: rawStore, var context = new VexConnectorContext(
SignatureVerifier: signatureVerifier, Since: null,
Normalizers: normalizerRouter, Settings: effectiveSettings,
Services: scopeProvider); RawSink: rawStore,
SignatureVerifier: signatureVerifier,
var documentCount = 0; Normalizers: normalizerRouter,
var claimCount = 0; Services: scopeProvider);
await foreach (var document in connector.FetchAsync(context, cancellationToken)) var documentCount = 0;
{ var claimCount = 0;
documentCount++;
await foreach (var document in connector.FetchAsync(context, cancellationToken))
var batch = await normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false); {
if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0) documentCount++;
{
claimCount += batch.Claims.Length; var batch = await normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false);
await claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false); if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0)
} {
} claimCount += batch.Claims.Length;
await claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false);
_logger.LogInformation( }
"Connector {ConnectorId} persisted {DocumentCount} document(s) and {ClaimCount} claim(s) this run.", }
connector.Id,
documentCount, _logger.LogInformation(
claimCount); "Connector {ConnectorId} persisted {DocumentCount} document(s) and {ClaimCount} claim(s) this run.",
} connector.Id,
} documentCount,
claimCount);
}
}

View File

@@ -1,204 +1,205 @@
using System.Text; using System.Text;
using StellaOps.Zastava.Core.Contracts; using System.Security.Cryptography;
using StellaOps.Zastava.Core.Hashing; using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Core.Serialization; using StellaOps.Zastava.Core.Hashing;
using StellaOps.Zastava.Core.Serialization;
namespace StellaOps.Zastava.Core.Tests.Serialization;
namespace StellaOps.Zastava.Core.Tests.Serialization;
public sealed class ZastavaCanonicalJsonSerializerTests
{ public sealed class ZastavaCanonicalJsonSerializerTests
[Fact] {
public void Serialize_RuntimeEventEnvelope_ProducesDeterministicOrdering() [Fact]
{ public void Serialize_RuntimeEventEnvelope_ProducesDeterministicOrdering()
var runtimeEvent = new RuntimeEvent {
{ var runtimeEvent = new RuntimeEvent
EventId = "evt-123", {
When = DateTimeOffset.Parse("2025-10-19T12:34:56Z"), EventId = "evt-123",
Kind = RuntimeEventKind.ContainerStart, When = DateTimeOffset.Parse("2025-10-19T12:34:56Z"),
Tenant = "tenant-01", Kind = RuntimeEventKind.ContainerStart,
Node = "node-a", Tenant = "tenant-01",
Runtime = new RuntimeEngine Node = "node-a",
{ Runtime = new RuntimeEngine
Engine = "containerd", {
Version = "1.7.19" Engine = "containerd",
}, Version = "1.7.19"
Workload = new RuntimeWorkload },
{ Workload = new RuntimeWorkload
Platform = "kubernetes", {
Namespace = "payments", Platform = "kubernetes",
Pod = "api-7c9fbbd8b7-ktd84", Namespace = "payments",
Container = "api", Pod = "api-7c9fbbd8b7-ktd84",
ContainerId = "containerd://abc", Container = "api",
ImageRef = "ghcr.io/acme/api@sha256:abcd", ContainerId = "containerd://abc",
Owner = new RuntimeWorkloadOwner ImageRef = "ghcr.io/acme/api@sha256:abcd",
{ Owner = new RuntimeWorkloadOwner
Kind = "Deployment", {
Name = "api" Kind = "Deployment",
} Name = "api"
}, }
Process = new RuntimeProcess },
{ Process = new RuntimeProcess
Pid = 12345, {
Entrypoint = new[] { "/entrypoint.sh", "--serve" }, Pid = 12345,
EntryTrace = new[] Entrypoint = new[] { "/entrypoint.sh", "--serve" },
{ EntryTrace = new[]
new RuntimeEntryTrace {
{ new RuntimeEntryTrace
File = "/entrypoint.sh", {
Line = 3, File = "/entrypoint.sh",
Op = "exec", Line = 3,
Target = "/usr/bin/python3" Op = "exec",
} Target = "/usr/bin/python3"
} }
}, }
LoadedLibraries = new[] },
{ LoadedLibraries = new[]
new RuntimeLoadedLibrary {
{ new RuntimeLoadedLibrary
Path = "/lib/x86_64-linux-gnu/libssl.so.3", {
Inode = 123456, Path = "/lib/x86_64-linux-gnu/libssl.so.3",
Sha256 = "abc123" Inode = 123456,
} Sha256 = "abc123"
}, }
Posture = new RuntimePosture },
{ Posture = new RuntimePosture
ImageSigned = true, {
SbomReferrer = "present", ImageSigned = true,
Attestation = new RuntimeAttestation SbomReferrer = "present",
{ Attestation = new RuntimeAttestation
Uuid = "rekor-uuid", {
Verified = true Uuid = "rekor-uuid",
} Verified = true
}, }
Delta = new RuntimeDelta },
{ Delta = new RuntimeDelta
BaselineImageDigest = "sha256:abcd", {
ChangedFiles = new[] { "/opt/app/server.py" }, BaselineImageDigest = "sha256:abcd",
NewBinaries = new[] ChangedFiles = new[] { "/opt/app/server.py" },
{ NewBinaries = new[]
new RuntimeNewBinary {
{ new RuntimeNewBinary
Path = "/usr/local/bin/helper", {
Sha256 = "def456" Path = "/usr/local/bin/helper",
} Sha256 = "def456"
} }
}, }
Evidence = new[] },
{ Evidence = new[]
new RuntimeEvidence {
{ new RuntimeEvidence
Signal = "procfs.maps", {
Value = "/lib/.../libssl.so.3@0x7f..." Signal = "procfs.maps",
} Value = "/lib/.../libssl.so.3@0x7f..."
}, }
Annotations = new Dictionary<string, string> },
{ Annotations = new Dictionary<string, string>
["source"] = "unit-test" {
} ["source"] = "unit-test"
}; }
};
var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent);
var json = ZastavaCanonicalJsonSerializer.Serialize(envelope); var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent);
var json = ZastavaCanonicalJsonSerializer.Serialize(envelope);
var expectedOrder = new[]
{ var expectedOrder = new[]
"\"schemaVersion\"", {
"\"event\"", "\"schemaVersion\"",
"\"eventId\"", "\"event\"",
"\"when\"", "\"eventId\"",
"\"kind\"", "\"when\"",
"\"tenant\"", "\"kind\"",
"\"node\"", "\"tenant\"",
"\"runtime\"", "\"node\"",
"\"engine\"", "\"runtime\"",
"\"version\"", "\"engine\"",
"\"workload\"", "\"version\"",
"\"platform\"", "\"workload\"",
"\"namespace\"", "\"platform\"",
"\"pod\"", "\"namespace\"",
"\"container\"", "\"pod\"",
"\"containerId\"", "\"container\"",
"\"imageRef\"", "\"containerId\"",
"\"owner\"", "\"imageRef\"",
"\"kind\"", "\"owner\"",
"\"name\"", "\"kind\"",
"\"process\"", "\"name\"",
"\"pid\"", "\"process\"",
"\"entrypoint\"", "\"pid\"",
"\"entryTrace\"", "\"entrypoint\"",
"\"loadedLibs\"", "\"entryTrace\"",
"\"posture\"", "\"loadedLibs\"",
"\"imageSigned\"", "\"posture\"",
"\"sbomReferrer\"", "\"imageSigned\"",
"\"attestation\"", "\"sbomReferrer\"",
"\"uuid\"", "\"attestation\"",
"\"verified\"", "\"uuid\"",
"\"delta\"", "\"verified\"",
"\"baselineImageDigest\"", "\"delta\"",
"\"changedFiles\"", "\"baselineImageDigest\"",
"\"newBinaries\"", "\"changedFiles\"",
"\"path\"", "\"newBinaries\"",
"\"sha256\"", "\"path\"",
"\"evidence\"", "\"sha256\"",
"\"signal\"", "\"evidence\"",
"\"value\"", "\"signal\"",
"\"annotations\"", "\"value\"",
"\"source\"" "\"annotations\"",
}; "\"source\""
};
var cursor = -1;
foreach (var token in expectedOrder) var cursor = -1;
{ foreach (var token in expectedOrder)
var position = json.IndexOf(token, cursor + 1, StringComparison.Ordinal); {
Assert.True(position > cursor, $"Property token {token} not found in the expected order."); var position = json.IndexOf(token, cursor + 1, StringComparison.Ordinal);
cursor = position; Assert.True(position > cursor, $"Property token {token} not found in the expected order.");
} cursor = position;
}
Assert.DoesNotContain(" ", json, StringComparison.Ordinal);
Assert.StartsWith("{\"schemaVersion\"", json, StringComparison.Ordinal); Assert.DoesNotContain(" ", json, StringComparison.Ordinal);
Assert.EndsWith("}}", json, StringComparison.Ordinal); Assert.StartsWith("{\"schemaVersion\"", json, StringComparison.Ordinal);
} Assert.EndsWith("}}", json, StringComparison.Ordinal);
}
[Fact]
public void ComputeMultihash_ProducesStableBase64UrlDigest() [Fact]
{ public void ComputeMultihash_ProducesStableBase64UrlDigest()
var decision = AdmissionDecisionEnvelope.Create( {
new AdmissionDecision var decision = AdmissionDecisionEnvelope.Create(
{ new AdmissionDecision
AdmissionId = "admission-123", {
Namespace = "payments", AdmissionId = "admission-123",
PodSpecDigest = "sha256:deadbeef", Namespace = "payments",
Images = new[] PodSpecDigest = "sha256:deadbeef",
{ Images = new[]
new AdmissionImageVerdict {
{ new AdmissionImageVerdict
Name = "ghcr.io/acme/api:1.2.3", {
Resolved = "ghcr.io/acme/api@sha256:abcd", Name = "ghcr.io/acme/api:1.2.3",
Signed = true, Resolved = "ghcr.io/acme/api@sha256:abcd",
HasSbomReferrers = true, Signed = true,
PolicyVerdict = PolicyVerdict.Pass, HasSbomReferrers = true,
Reasons = Array.Empty<string>(), PolicyVerdict = PolicyVerdict.Pass,
Rekor = new AdmissionRekorEvidence Reasons = Array.Empty<string>(),
{ Rekor = new AdmissionRekorEvidence
Uuid = "xyz", {
Verified = true Uuid = "xyz",
} Verified = true
} }
}, }
Decision = AdmissionDecisionOutcome.Allow, },
TtlSeconds = 300 Decision = AdmissionDecisionOutcome.Allow,
}, TtlSeconds = 300
ZastavaContractVersions.AdmissionDecision); },
ZastavaContractVersions.AdmissionDecision);
var canonicalJson = ZastavaCanonicalJsonSerializer.Serialize(decision);
var expectedDigestBytes = SHA256.HashData(Encoding.UTF8.GetBytes(canonicalJson)); var canonicalJson = ZastavaCanonicalJsonSerializer.Serialize(decision);
var expected = $"sha256-{Convert.ToBase64String(expectedDigestBytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}"; var expectedDigestBytes = SHA256.HashData(Encoding.UTF8.GetBytes(canonicalJson));
var expected = $"sha256-{Convert.ToBase64String(expectedDigestBytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}";
var hash = ZastavaHashing.ComputeMultihash(decision);
var hash = ZastavaHashing.ComputeMultihash(decision);
Assert.Equal(expected, hash);
Assert.Equal(expected, hash);
var sha512 = ZastavaHashing.ComputeMultihash(Encoding.UTF8.GetBytes(canonicalJson), "sha512");
Assert.StartsWith("sha512-", sha512, StringComparison.Ordinal); var sha512 = ZastavaHashing.ComputeMultihash(Encoding.UTF8.GetBytes(canonicalJson), "sha512");
} Assert.StartsWith("sha512-", sha512, StringComparison.Ordinal);
} }
}

View File

@@ -104,8 +104,8 @@ public static class ZastavaContractVersions
/// <summary> /// <summary>
/// Canonical string representation (schema@vMajor.Minor). /// Canonical string representation (schema@vMajor.Minor).
/// </summary> /// </summary>
public override string ToString() public override string ToString()
=> $"{Schema}@v{Version.ToString(2, CultureInfo.InvariantCulture)}"; => $"{Schema}@v{Version.ToString(2)}";
/// <summary> /// <summary>
/// Determines whether a remote contract is compatible with the local definition. /// Determines whether a remote contract is compatible with the local definition.

View File

@@ -1,4 +1,13 @@
namespace StellaOps.Zastava.Core.Serialization; using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
using StellaOps.Zastava.Core.Contracts;
namespace StellaOps.Zastava.Core.Serialization;
/// <summary> /// <summary>
/// Deterministic serializer used for runtime/admission contracts. /// Deterministic serializer used for runtime/admission contracts.