stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcConnector.cs
View File

@@ -9,7 +9,7 @@ using System.Text.Json.Serialization;
using System.Threading;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
@@ -338,7 +338,7 @@ public sealed class CertCcConnector : IFeedConnector
var dto = CertCcNoteParser.Parse(noteBytes, vendorsBytes, vulsBytes, vendorStatusesBytes);
var json = JsonSerializer.Serialize(dto, DtoSerializerOptions);
var payload = StellaOps.Concelier.Bson.BsonDocument.Parse(json);
var payload = StellaOps.Concelier.Documents.DocumentObject.Parse(json);
_diagnostics.ParseSuccess(
dto.Vendors.Count,
@@ -678,7 +678,7 @@ public sealed class CertCcConnector : IFeedConnector
private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken)
{
var completedAt = _timeProvider.GetUtcNow();
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false);
await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToDocumentObject(), completedAt, cancellationToken).ConfigureAwait(false);
}
private sealed class NoteDocumentGroup

42
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcConnectorPlugin.cs
View File

@@ -1,21 +1,21 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-cc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertCcConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertCcConnector>();
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcConnectorPlugin : IConnectorPlugin
{
public const string SourceName = "cert-cc";
public string Name => SourceName;
public bool IsAvailable(IServiceProvider services)
=> services.GetService<CertCcConnector>() is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return services.GetRequiredService<CertCcConnector>();
}
}

100
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs
View File

@@ -1,50 +1,50 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertCc.Configuration;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-cc";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertCcConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertCcFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.CertCc.Configuration;
namespace StellaOps.Concelier.Connector.CertCc;
public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:cert-cc";
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddCertCcConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
services.AddTransient<CertCcFetchJob>();
services.PostConfigure<JobSchedulerOptions>(options =>
{
EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob));
});
return services;
}
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
{
if (options.Definitions.ContainsKey(kind))
{
return;
}
options.Definitions[kind] = new JobDefinition(
kind,
jobType,
options.DefaultTimeout,
options.DefaultLeaseDuration,
CronExpression: null,
Enabled: true);
}
}

74
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/CertCcServiceCollectionExtensions.cs
View File

@@ -1,37 +1,37 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertCc;
public static class CertCcServiceCollectionExtensions
{
public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action<CertCcOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertCcOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertCcOptions>>().Value;
clientOptions.BaseAddress = options.BaseApiUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseApiUri.Host);
});
services.TryAddSingleton<CertCcSummaryPlanner>();
services.TryAddSingleton<CertCcDiagnostics>();
services.AddTransient<CertCcConnector>();
return services;
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.CertCc.Internal;
using StellaOps.Concelier.Connector.Common.Http;
namespace StellaOps.Concelier.Connector.CertCc;
public static class CertCcServiceCollectionExtensions
{
public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action<CertCcOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<CertCcOptions>()
.Configure(configure)
.PostConfigure(static options => options.Validate());
services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<CertCcOptions>>().Value;
clientOptions.BaseAddress = options.BaseApiUri;
clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0";
clientOptions.Timeout = TimeSpan.FromSeconds(20);
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseApiUri.Host);
});
services.TryAddSingleton<CertCcSummaryPlanner>();
services.TryAddSingleton<CertCcDiagnostics>();
services.AddTransient<CertCcConnector>();
return services;
}
}

158
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Configuration/CertCcOptions.cs
View File

@@ -1,79 +1,79 @@
using System;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Configuration;
/// <summary>
/// Connector options governing CERT/CC fetch cadence and API endpoints.
/// </summary>
public sealed class CertCcOptions
{
public const string HttpClientName = "certcc";
/// <summary>
/// Root URI for the VINCE Vulnerability Notes API (must end with a slash).
/// </summary>
public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
/// <summary>
/// Sliding window settings controlling which summary endpoints are requested.
/// </summary>
public TimeWindowCursorOptions SummaryWindow { get; set; } = new()
{
WindowSize = TimeSpan.FromDays(30),
Overlap = TimeSpan.FromDays(3),
InitialBackfill = TimeSpan.FromDays(365),
MinimumWindowSize = TimeSpan.FromDays(1),
};
/// <summary>
/// Maximum number of monthly summary endpoints to request in a single plan.
/// </summary>
public int MaxMonthlySummaries { get; set; } = 6;
/// <summary>
/// Maximum number of vulnerability notes (detail bundles) to process per fetch pass.
/// </summary>
public int MaxNotesPerFetch { get; set; } = 25;
/// <summary>
/// Optional delay inserted between successive detail requests to respect upstream throttling.
/// </summary>
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
/// <summary>
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
/// </summary>
public bool EnableDetailMapping { get; set; } = true;
public void Validate()
{
if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI.");
}
if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash.");
}
SummaryWindow ??= new TimeWindowCursorOptions();
SummaryWindow.EnsureValid();
if (MaxMonthlySummaries <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive.");
}
if (MaxNotesPerFetch <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive.");
}
if (DetailRequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative.");
}
}
}
using System;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Configuration;
/// <summary>
/// Connector options governing CERT/CC fetch cadence and API endpoints.
/// </summary>
public sealed class CertCcOptions
{
public const string HttpClientName = "certcc";
/// <summary>
/// Root URI for the VINCE Vulnerability Notes API (must end with a slash).
/// </summary>
public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute);
/// <summary>
/// Sliding window settings controlling which summary endpoints are requested.
/// </summary>
public TimeWindowCursorOptions SummaryWindow { get; set; } = new()
{
WindowSize = TimeSpan.FromDays(30),
Overlap = TimeSpan.FromDays(3),
InitialBackfill = TimeSpan.FromDays(365),
MinimumWindowSize = TimeSpan.FromDays(1),
};
/// <summary>
/// Maximum number of monthly summary endpoints to request in a single plan.
/// </summary>
public int MaxMonthlySummaries { get; set; } = 6;
/// <summary>
/// Maximum number of vulnerability notes (detail bundles) to process per fetch pass.
/// </summary>
public int MaxNotesPerFetch { get; set; } = 25;
/// <summary>
/// Optional delay inserted between successive detail requests to respect upstream throttling.
/// </summary>
public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100);
/// <summary>
/// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging.
/// </summary>
public bool EnableDetailMapping { get; set; } = true;
public void Validate()
{
if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri)
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI.");
}
if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash.");
}
SummaryWindow ??= new TimeWindowCursorOptions();
SummaryWindow.EnsureValid();
if (MaxMonthlySummaries <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive.");
}
if (MaxNotesPerFetch <= 0)
{
throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive.");
}
if (DetailRequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative.");
}
}
}

46
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs
View File

@@ -1,4 +1,4 @@
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
@@ -22,18 +22,18 @@ internal sealed record CertCcCursor(
EmptyGuidArray,
null);
public BsonDocument ToBsonDocument()
public DocumentObject ToDocumentObject()
{
var document = new BsonDocument();
var document = new DocumentObject();
var summary = new BsonDocument();
var summary = new DocumentObject();
SummaryState.WriteTo(summary, "start", "end");
document["summary"] = summary;
document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString()));
document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note));
document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString()));
document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString()));
document["pendingSummaries"] = new DocumentArray(PendingSummaries.Select(static id => id.ToString()));
document["pendingNotes"] = new DocumentArray(PendingNotes.Select(static note => note));
document["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(static id => id.ToString()));
document["pendingMappings"] = new DocumentArray(PendingMappings.Select(static id => id.ToString()));
if (LastRun.HasValue)
{
@@ -43,7 +43,7 @@ internal sealed record CertCcCursor(
return document;
}
public static CertCcCursor FromBson(BsonDocument? document)
public static CertCcCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
@@ -51,9 +51,9 @@ internal sealed record CertCcCursor(
}
TimeWindowCursorState summaryState = TimeWindowCursorState.Empty;
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument)
if (document.TryGetValue("summary", out var summaryValue) && summaryValue is DocumentObject summaryDocument)
{
summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end");
summaryState = TimeWindowCursorState.FromDocumentObject(summaryDocument, "start", "end");
}
var pendingSummaries = ReadGuidArray(document, "pendingSummaries");
@@ -64,10 +64,10 @@ internal sealed record CertCcCursor(
DateTimeOffset? lastRun = null;
if (document.TryGetValue("lastRun", out var lastRunValue))
{
lastRun = lastRunValue.BsonType switch
lastRun = lastRunValue.DocumentType switch
{
BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
DocumentType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
@@ -93,9 +93,9 @@ internal sealed record CertCcCursor(
public CertCcCursor WithLastRun(DateTimeOffset? timestamp)
=> this with { LastRun = timestamp };
private static Guid[] ReadGuidArray(BsonDocument document, string field)
private static Guid[] ReadGuidArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyGuidArray;
}
@@ -112,9 +112,9 @@ internal sealed record CertCcCursor(
return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray();
}
private static string[] ReadStringArray(BsonDocument document, string field)
private static string[] ReadStringArray(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0)
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array || array.Count == 0)
{
return EmptyStringArray;
}
@@ -124,10 +124,10 @@ internal sealed record CertCcCursor(
{
switch (element)
{
case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
case DocumentString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString):
results.Add(bsonString.AsString.Trim());
break;
case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
case DocumentObject bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString:
results.Add(inner.AsString.Trim());
break;
}
@@ -142,14 +142,14 @@ internal sealed record CertCcCursor(
.ToArray();
}
private static bool TryReadGuid(BsonValue value, out Guid guid)
private static bool TryReadGuid(DocumentValue value, out Guid guid)
{
if (value is BsonString bsonString && Guid.TryParse(bsonString.AsString, out guid))
if (value is DocumentString bsonString && Guid.TryParse(bsonString.AsString, out guid))
{
return true;
}
if (value is BsonBinaryData binary)
if (value is DocumentBinaryData binary)
{
try
{

428
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcDiagnostics.cs
View File

@@ -1,214 +1,214 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
/// </summary>
public sealed class CertCcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _planWindows;
private readonly Counter<long> _planRequests;
private readonly Histogram<double> _planWindowDays;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchUnchanged;
private readonly Counter<long> _summaryFetchFailures;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchUnchanged;
private readonly Counter<long> _detailFetchMissing;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseVendorCount;
private readonly Histogram<long> _parseStatusCount;
private readonly Histogram<long> _parseVulnerabilityCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAffectedPackageCount;
private readonly Histogram<long> _mapNormalizedVersionCount;
public CertCcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_planWindows = _meter.CreateCounter<long>(
name: "certcc.plan.windows",
unit: "windows",
description: "Number of summary planning windows evaluated.");
_planRequests = _meter.CreateCounter<long>(
name: "certcc.plan.requests",
unit: "requests",
description: "Total CERT/CC summary endpoints queued by the planner.");
_planWindowDays = _meter.CreateHistogram<double>(
name: "certcc.plan.window_days",
unit: "day",
description: "Duration of each planning window in days.");
_summaryFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.attempts",
unit: "operations",
description: "Number of VINCE summary fetch attempts.");
_summaryFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.success",
unit: "operations",
description: "Number of VINCE summary fetches persisted to storage.");
_summaryFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.not_modified",
unit: "operations",
description: "Number of VINCE summary fetches returning HTTP 304.");
_summaryFetchFailures = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.failures",
unit: "operations",
description: "Number of VINCE summary fetches that failed after retries.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.attempts",
unit: "operations",
description: "Number of VINCE detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.success",
unit: "operations",
description: "Number of VINCE detail fetches that returned payloads.");
_detailFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.unchanged",
unit: "operations",
description: "Number of VINCE detail fetches returning HTTP 304.");
_detailFetchMissing = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.missing",
unit: "operations",
description: "Number of optional VINCE detail endpoints missing but tolerated.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.failures",
unit: "operations",
description: "Number of VINCE detail fetches that failed after retries.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certcc.parse.success",
unit: "documents",
description: "Number of VINCE note bundles parsed into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certcc.parse.failures",
unit: "documents",
description: "Number of VINCE note bundles that failed to parse.");
_parseVendorCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vendors.count",
unit: "vendors",
description: "Distribution of vendor statements per VINCE note.");
_parseStatusCount = _meter.CreateHistogram<long>(
name: "certcc.parse.statuses.count",
unit: "entries",
description: "Distribution of vendor status entries per VINCE note.");
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vulnerabilities.count",
unit: "entries",
description: "Distribution of vulnerability records per VINCE note.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certcc.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certcc.map.failures",
unit: "advisories",
description: "Number of CERT/CC advisory mapping attempts that failed.");
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
name: "certcc.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per CERT/CC advisory.");
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
name: "certcc.map.normalized_versions.count",
unit: "rules",
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
}
public void PlanEvaluated(TimeWindow window, int requestCount)
{
_planWindows.Add(1);
if (requestCount > 0)
{
_planRequests.Add(requestCount);
}
var duration = window.Duration;
if (duration > TimeSpan.Zero)
{
_planWindowDays.Record(duration.TotalDays);
}
}
public void SummaryFetchAttempt(CertCcSummaryScope scope)
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
public void SummaryFetchSuccess(CertCcSummaryScope scope)
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
public void SummaryFetchFailure(CertCcSummaryScope scope)
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
public void DetailFetchAttempt(string endpoint)
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
public void DetailFetchSuccess(string endpoint)
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
public void DetailFetchUnchanged(string endpoint)
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
public void DetailFetchMissing(string endpoint)
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
public void DetailFetchFailure(string endpoint)
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
{
_parseSuccess.Add(1);
if (vendorCount >= 0)
{
_parseVendorCount.Record(vendorCount);
}
if (statusCount >= 0)
{
_parseStatusCount.Record(statusCount);
}
if (vulnerabilityCount >= 0)
{
_parseVulnerabilityCount.Record(vulnerabilityCount);
}
}
public void ParseFailure()
=> _parseFailures.Add(1);
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
{
_mapSuccess.Add(1);
if (affectedPackageCount >= 0)
{
_mapAffectedPackageCount.Record(affectedPackageCount);
}
if (normalizedVersionCount >= 0)
{
_mapNormalizedVersionCount.Record(normalizedVersionCount);
}
}
public void MapFailure()
=> _mapFailures.Add(1);
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
=> new("scope", scope.ToString().ToLowerInvariant());
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Emits CERT/CC-specific telemetry for summary planning and fetch activity.
/// </summary>
public sealed class CertCcDiagnostics : IDisposable
{
private const string MeterName = "StellaOps.Concelier.Connector.CertCc";
private const string MeterVersion = "1.0.0";
private readonly Meter _meter;
private readonly Counter<long> _planWindows;
private readonly Counter<long> _planRequests;
private readonly Histogram<double> _planWindowDays;
private readonly Counter<long> _summaryFetchAttempts;
private readonly Counter<long> _summaryFetchSuccess;
private readonly Counter<long> _summaryFetchUnchanged;
private readonly Counter<long> _summaryFetchFailures;
private readonly Counter<long> _detailFetchAttempts;
private readonly Counter<long> _detailFetchSuccess;
private readonly Counter<long> _detailFetchUnchanged;
private readonly Counter<long> _detailFetchMissing;
private readonly Counter<long> _detailFetchFailures;
private readonly Counter<long> _parseSuccess;
private readonly Counter<long> _parseFailures;
private readonly Histogram<long> _parseVendorCount;
private readonly Histogram<long> _parseStatusCount;
private readonly Histogram<long> _parseVulnerabilityCount;
private readonly Counter<long> _mapSuccess;
private readonly Counter<long> _mapFailures;
private readonly Histogram<long> _mapAffectedPackageCount;
private readonly Histogram<long> _mapNormalizedVersionCount;
public CertCcDiagnostics()
{
_meter = new Meter(MeterName, MeterVersion);
_planWindows = _meter.CreateCounter<long>(
name: "certcc.plan.windows",
unit: "windows",
description: "Number of summary planning windows evaluated.");
_planRequests = _meter.CreateCounter<long>(
name: "certcc.plan.requests",
unit: "requests",
description: "Total CERT/CC summary endpoints queued by the planner.");
_planWindowDays = _meter.CreateHistogram<double>(
name: "certcc.plan.window_days",
unit: "day",
description: "Duration of each planning window in days.");
_summaryFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.attempts",
unit: "operations",
description: "Number of VINCE summary fetch attempts.");
_summaryFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.success",
unit: "operations",
description: "Number of VINCE summary fetches persisted to storage.");
_summaryFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.not_modified",
unit: "operations",
description: "Number of VINCE summary fetches returning HTTP 304.");
_summaryFetchFailures = _meter.CreateCounter<long>(
name: "certcc.summary.fetch.failures",
unit: "operations",
description: "Number of VINCE summary fetches that failed after retries.");
_detailFetchAttempts = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.attempts",
unit: "operations",
description: "Number of VINCE detail fetch attempts.");
_detailFetchSuccess = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.success",
unit: "operations",
description: "Number of VINCE detail fetches that returned payloads.");
_detailFetchUnchanged = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.unchanged",
unit: "operations",
description: "Number of VINCE detail fetches returning HTTP 304.");
_detailFetchMissing = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.missing",
unit: "operations",
description: "Number of optional VINCE detail endpoints missing but tolerated.");
_detailFetchFailures = _meter.CreateCounter<long>(
name: "certcc.detail.fetch.failures",
unit: "operations",
description: "Number of VINCE detail fetches that failed after retries.");
_parseSuccess = _meter.CreateCounter<long>(
name: "certcc.parse.success",
unit: "documents",
description: "Number of VINCE note bundles parsed into DTOs.");
_parseFailures = _meter.CreateCounter<long>(
name: "certcc.parse.failures",
unit: "documents",
description: "Number of VINCE note bundles that failed to parse.");
_parseVendorCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vendors.count",
unit: "vendors",
description: "Distribution of vendor statements per VINCE note.");
_parseStatusCount = _meter.CreateHistogram<long>(
name: "certcc.parse.statuses.count",
unit: "entries",
description: "Distribution of vendor status entries per VINCE note.");
_parseVulnerabilityCount = _meter.CreateHistogram<long>(
name: "certcc.parse.vulnerabilities.count",
unit: "entries",
description: "Distribution of vulnerability records per VINCE note.");
_mapSuccess = _meter.CreateCounter<long>(
name: "certcc.map.success",
unit: "advisories",
description: "Number of canonical advisories emitted by the CERT/CC mapper.");
_mapFailures = _meter.CreateCounter<long>(
name: "certcc.map.failures",
unit: "advisories",
description: "Number of CERT/CC advisory mapping attempts that failed.");
_mapAffectedPackageCount = _meter.CreateHistogram<long>(
name: "certcc.map.affected.count",
unit: "packages",
description: "Distribution of affected packages emitted per CERT/CC advisory.");
_mapNormalizedVersionCount = _meter.CreateHistogram<long>(
name: "certcc.map.normalized_versions.count",
unit: "rules",
description: "Distribution of normalized version rules emitted per CERT/CC advisory.");
}
public void PlanEvaluated(TimeWindow window, int requestCount)
{
_planWindows.Add(1);
if (requestCount > 0)
{
_planRequests.Add(requestCount);
}
var duration = window.Duration;
if (duration > TimeSpan.Zero)
{
_planWindowDays.Record(duration.TotalDays);
}
}
public void SummaryFetchAttempt(CertCcSummaryScope scope)
=> _summaryFetchAttempts.Add(1, ScopeTag(scope));
public void SummaryFetchSuccess(CertCcSummaryScope scope)
=> _summaryFetchSuccess.Add(1, ScopeTag(scope));
public void SummaryFetchUnchanged(CertCcSummaryScope scope)
=> _summaryFetchUnchanged.Add(1, ScopeTag(scope));
public void SummaryFetchFailure(CertCcSummaryScope scope)
=> _summaryFetchFailures.Add(1, ScopeTag(scope));
public void DetailFetchAttempt(string endpoint)
=> _detailFetchAttempts.Add(1, EndpointTag(endpoint));
public void DetailFetchSuccess(string endpoint)
=> _detailFetchSuccess.Add(1, EndpointTag(endpoint));
public void DetailFetchUnchanged(string endpoint)
=> _detailFetchUnchanged.Add(1, EndpointTag(endpoint));
public void DetailFetchMissing(string endpoint)
=> _detailFetchMissing.Add(1, EndpointTag(endpoint));
public void DetailFetchFailure(string endpoint)
=> _detailFetchFailures.Add(1, EndpointTag(endpoint));
public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount)
{
_parseSuccess.Add(1);
if (vendorCount >= 0)
{
_parseVendorCount.Record(vendorCount);
}
if (statusCount >= 0)
{
_parseStatusCount.Record(statusCount);
}
if (vulnerabilityCount >= 0)
{
_parseVulnerabilityCount.Record(vulnerabilityCount);
}
}
public void ParseFailure()
=> _parseFailures.Add(1);
public void MapSuccess(int affectedPackageCount, int normalizedVersionCount)
{
_mapSuccess.Add(1);
if (affectedPackageCount >= 0)
{
_mapAffectedPackageCount.Record(affectedPackageCount);
}
if (normalizedVersionCount >= 0)
{
_mapNormalizedVersionCount.Record(normalizedVersionCount);
}
}
public void MapFailure()
=> _mapFailures.Add(1);
private static KeyValuePair<string, object?> ScopeTag(CertCcSummaryScope scope)
=> new("scope", scope.ToString().ToLowerInvariant());
private static KeyValuePair<string, object?> EndpointTag(string endpoint)
=> new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant());
public void Dispose() => _meter.Dispose();
}

1214
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcMapper.cs
View File

File diff suppressed because it is too large Load Diff

194
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteDto.cs
View File

@@ -1,97 +1,97 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal sealed record CertCcNoteDto(
CertCcNoteMetadata Metadata,
IReadOnlyList<CertCcVendorDto> Vendors,
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
{
public static CertCcNoteDto Empty { get; } = new(
CertCcNoteMetadata.Empty,
Array.Empty<CertCcVendorDto>(),
Array.Empty<CertCcVendorStatusDto>(),
Array.Empty<CertCcVulnerabilityDto>());
}
internal sealed record CertCcNoteMetadata(
string? VuId,
string IdNumber,
string Title,
string? Overview,
string? Summary,
DateTimeOffset? Published,
DateTimeOffset? Updated,
DateTimeOffset? Created,
int? Revision,
IReadOnlyList<string> CveIds,
IReadOnlyList<string> PublicUrls,
string? PrimaryUrl)
{
public static CertCcNoteMetadata Empty { get; } = new(
VuId: null,
IdNumber: string.Empty,
Title: string.Empty,
Overview: null,
Summary: null,
Published: null,
Updated: null,
Created: null,
Revision: null,
CveIds: Array.Empty<string>(),
PublicUrls: Array.Empty<string>(),
PrimaryUrl: null);
}
internal sealed record CertCcVendorDto(
string Vendor,
DateTimeOffset? ContactDate,
DateTimeOffset? StatementDate,
DateTimeOffset? Updated,
string? Statement,
string? Addendum,
IReadOnlyList<string> References)
{
public static CertCcVendorDto Empty { get; } = new(
Vendor: string.Empty,
ContactDate: null,
StatementDate: null,
Updated: null,
Statement: null,
Addendum: null,
References: Array.Empty<string>());
}
internal sealed record CertCcVendorStatusDto(
string Vendor,
string CveId,
string Status,
string? Statement,
IReadOnlyList<string> References,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVendorStatusDto Empty { get; } = new(
Vendor: string.Empty,
CveId: string.Empty,
Status: string.Empty,
Statement: null,
References: Array.Empty<string>(),
DateAdded: null,
DateUpdated: null);
}
internal sealed record CertCcVulnerabilityDto(
string CveId,
string? Description,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVulnerabilityDto Empty { get; } = new(
CveId: string.Empty,
Description: null,
DateAdded: null,
DateUpdated: null);
}
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal sealed record CertCcNoteDto(
CertCcNoteMetadata Metadata,
IReadOnlyList<CertCcVendorDto> Vendors,
IReadOnlyList<CertCcVendorStatusDto> VendorStatuses,
IReadOnlyList<CertCcVulnerabilityDto> Vulnerabilities)
{
public static CertCcNoteDto Empty { get; } = new(
CertCcNoteMetadata.Empty,
Array.Empty<CertCcVendorDto>(),
Array.Empty<CertCcVendorStatusDto>(),
Array.Empty<CertCcVulnerabilityDto>());
}
internal sealed record CertCcNoteMetadata(
string? VuId,
string IdNumber,
string Title,
string? Overview,
string? Summary,
DateTimeOffset? Published,
DateTimeOffset? Updated,
DateTimeOffset? Created,
int? Revision,
IReadOnlyList<string> CveIds,
IReadOnlyList<string> PublicUrls,
string? PrimaryUrl)
{
public static CertCcNoteMetadata Empty { get; } = new(
VuId: null,
IdNumber: string.Empty,
Title: string.Empty,
Overview: null,
Summary: null,
Published: null,
Updated: null,
Created: null,
Revision: null,
CveIds: Array.Empty<string>(),
PublicUrls: Array.Empty<string>(),
PrimaryUrl: null);
}
internal sealed record CertCcVendorDto(
string Vendor,
DateTimeOffset? ContactDate,
DateTimeOffset? StatementDate,
DateTimeOffset? Updated,
string? Statement,
string? Addendum,
IReadOnlyList<string> References)
{
public static CertCcVendorDto Empty { get; } = new(
Vendor: string.Empty,
ContactDate: null,
StatementDate: null,
Updated: null,
Statement: null,
Addendum: null,
References: Array.Empty<string>());
}
internal sealed record CertCcVendorStatusDto(
string Vendor,
string CveId,
string Status,
string? Statement,
IReadOnlyList<string> References,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVendorStatusDto Empty { get; } = new(
Vendor: string.Empty,
CveId: string.Empty,
Status: string.Empty,
Statement: null,
References: Array.Empty<string>(),
DateAdded: null,
DateUpdated: null);
}
internal sealed record CertCcVulnerabilityDto(
string CveId,
string? Description,
DateTimeOffset? DateAdded,
DateTimeOffset? DateUpdated)
{
public static CertCcVulnerabilityDto Empty { get; } = new(
CveId: string.Empty,
Description: null,
DateAdded: null,
DateUpdated: null);
}

1078
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteParser.cs
View File

File diff suppressed because it is too large Load Diff

216
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryParser.cs
View File

@@ -1,108 +1,108 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcSummaryParser
{
public static IReadOnlyList<string> ParseNotes(byte[] payload)
{
if (payload is null || payload.Length == 0)
{
return Array.Empty<string>();
}
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip,
});
var notesElement = document.RootElement.ValueKind switch
{
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
JsonValueKind.Array => document.RootElement,
JsonValueKind.Null or JsonValueKind.Undefined => default,
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
};
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
{
return Array.Empty<string>();
}
var results = new List<string>(notesElement.GetArrayLength());
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var element in notesElement.EnumerateArray())
{
var token = ExtractToken(element);
if (string.IsNullOrWhiteSpace(token))
{
continue;
}
var normalized = token.Trim();
var dedupKey = CreateDedupKey(normalized);
if (seen.Add(dedupKey))
{
results.Add(normalized);
}
}
return results.Count == 0 ? Array.Empty<string>() : results;
}
private static string CreateDedupKey(string token)
{
var digits = string.Concat(token.Where(char.IsDigit));
return digits.Length > 0
? digits
: token.Trim().ToUpperInvariant();
}
private static string? ExtractToken(JsonElement element)
{
return element.ValueKind switch
{
JsonValueKind.String => element.GetString(),
JsonValueKind.Number => element.TryGetInt64(out var number)
? number.ToString(CultureInfo.InvariantCulture)
: element.GetRawText(),
JsonValueKind.Object => ExtractFromObject(element),
_ => null,
};
}
private static string? ExtractFromObject(JsonElement element)
{
foreach (var propertyName in PropertyCandidates)
{
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
{
var value = property.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
return null;
}
private static readonly string[] PropertyCandidates =
{
"note",
"notes",
"id",
"idnumber",
"noteId",
"vu",
"vuid",
"vuId",
};
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcSummaryParser
{
public static IReadOnlyList<string> ParseNotes(byte[] payload)
{
if (payload is null || payload.Length == 0)
{
return Array.Empty<string>();
}
using var document = JsonDocument.Parse(payload, new JsonDocumentOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip,
});
var notesElement = document.RootElement.ValueKind switch
{
JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes,
JsonValueKind.Array => document.RootElement,
JsonValueKind.Null or JsonValueKind.Undefined => default,
_ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."),
};
if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0)
{
return Array.Empty<string>();
}
var results = new List<string>(notesElement.GetArrayLength());
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var element in notesElement.EnumerateArray())
{
var token = ExtractToken(element);
if (string.IsNullOrWhiteSpace(token))
{
continue;
}
var normalized = token.Trim();
var dedupKey = CreateDedupKey(normalized);
if (seen.Add(dedupKey))
{
results.Add(normalized);
}
}
return results.Count == 0 ? Array.Empty<string>() : results;
}
private static string CreateDedupKey(string token)
{
var digits = string.Concat(token.Where(char.IsDigit));
return digits.Length > 0
? digits
: token.Trim().ToUpperInvariant();
}
private static string? ExtractToken(JsonElement element)
{
return element.ValueKind switch
{
JsonValueKind.String => element.GetString(),
JsonValueKind.Number => element.TryGetInt64(out var number)
? number.ToString(CultureInfo.InvariantCulture)
: element.GetRawText(),
JsonValueKind.Object => ExtractFromObject(element),
_ => null,
};
}
private static string? ExtractFromObject(JsonElement element)
{
foreach (var propertyName in PropertyCandidates)
{
if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String)
{
var value = property.GetString();
if (!string.IsNullOrWhiteSpace(value))
{
return value;
}
}
}
return null;
}
private static readonly string[] PropertyCandidates =
{
"note",
"notes",
"id",
"idnumber",
"noteId",
"vu",
"vuid",
"vuId",
};
}

44
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlan.cs
View File

@@ -1,22 +1,22 @@
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
public sealed record CertCcSummaryPlan(
TimeWindow Window,
IReadOnlyList<CertCcSummaryRequest> Requests,
TimeWindowCursorState NextState);
public enum CertCcSummaryScope
{
Monthly,
Yearly,
}
public sealed record CertCcSummaryRequest(
Uri Uri,
CertCcSummaryScope Scope,
int Year,
int? Month);
using System;
using System.Collections.Generic;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
public sealed record CertCcSummaryPlan(
TimeWindow Window,
IReadOnlyList<CertCcSummaryRequest> Requests,
TimeWindowCursorState NextState);
public enum CertCcSummaryScope
{
Monthly,
Yearly,
}
public sealed record CertCcSummaryRequest(
Uri Uri,
CertCcSummaryScope Scope,
int Year,
int? Month);

192
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlanner.cs
View File

@@ -1,96 +1,96 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
/// </summary>
public sealed class CertCcSummaryPlanner
{
private readonly CertCcOptions _options;
private readonly TimeProvider _timeProvider;
public CertCcSummaryPlanner(
IOptions<CertCcOptions> options,
TimeProvider? timeProvider = null)
{
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
}
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
{
var now = _timeProvider.GetUtcNow();
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
var months = EnumerateYearMonths(window.Start, window.End)
.Take(_options.MaxMonthlySummaries)
.ToArray();
if (months.Length == 0)
{
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
}
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
foreach (var month in months)
{
requests.Add(new CertCcSummaryRequest(
BuildMonthlyUri(month.Year, month.Month),
CertCcSummaryScope.Monthly,
month.Year,
month.Month));
}
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
{
requests.Add(new CertCcSummaryRequest(
BuildYearlyUri(year),
CertCcSummaryScope.Yearly,
year,
Month: null));
}
return new CertCcSummaryPlan(window, requests, nextState);
}
private Uri BuildMonthlyUri(int year, int month)
{
var path = $"{year:D4}/{month:D2}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private Uri BuildYearlyUri(int year)
{
var path = $"{year:D4}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
{
if (end <= start)
{
yield break;
}
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
{
limit = limit.AddMonths(1);
}
while (cursor < limit)
{
yield return (cursor.Year, cursor.Month);
cursor = cursor.AddMonths(1);
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.CertCc.Configuration;
using StellaOps.Concelier.Connector.Common.Cursors;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
/// <summary>
/// Computes which CERT/CC summary endpoints should be fetched for the next export window.
/// </summary>
public sealed class CertCcSummaryPlanner
{
private readonly CertCcOptions _options;
private readonly TimeProvider _timeProvider;
public CertCcSummaryPlanner(
IOptions<CertCcOptions> options,
TimeProvider? timeProvider = null)
{
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
}
public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state)
{
var now = _timeProvider.GetUtcNow();
var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow);
var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window);
var months = EnumerateYearMonths(window.Start, window.End)
.Take(_options.MaxMonthlySummaries)
.ToArray();
if (months.Length == 0)
{
return new CertCcSummaryPlan(window, Array.Empty<CertCcSummaryRequest>(), nextState);
}
var requests = new List<CertCcSummaryRequest>(months.Length * 2);
foreach (var month in months)
{
requests.Add(new CertCcSummaryRequest(
BuildMonthlyUri(month.Year, month.Month),
CertCcSummaryScope.Monthly,
month.Year,
month.Month));
}
foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year))
{
requests.Add(new CertCcSummaryRequest(
BuildYearlyUri(year),
CertCcSummaryScope.Yearly,
year,
Month: null));
}
return new CertCcSummaryPlan(window, requests, nextState);
}
private Uri BuildMonthlyUri(int year, int month)
{
var path = $"{year:D4}/{month:D2}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private Uri BuildYearlyUri(int year)
{
var path = $"{year:D4}/summary/";
return new Uri(_options.BaseApiUri, path);
}
private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end)
{
if (end <= start)
{
yield break;
}
var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc);
var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc);
if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero)
{
limit = limit.AddMonths(1);
}
while (cursor < limit)
{
yield return (cursor.Year, cursor.Month);
cursor = cursor.AddMonths(1);
}
}
}

470
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Internal/CertCcVendorStatementParser.cs
View File

@@ -1,235 +1,235 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcVendorStatementParser
{
private static readonly string[] PairSeparators =
{
"\t",
" - ",
" – ",
" — ",
" : ",
": ",
" :",
":",
};
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
{
if (string.IsNullOrWhiteSpace(statement))
{
return Array.Empty<CertCcVendorPatch>();
}
var patches = new List<CertCcVendorPatch>();
var lines = statement
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (var rawLine in lines)
{
var line = rawLine.Trim();
if (line.Length == 0)
{
continue;
}
line = TrimBulletPrefix(line);
if (line.Length == 0)
{
continue;
}
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
{
continue;
}
var versions = ExtractVersions(versionSegment);
if (versions.Count == 0)
{
continue;
}
var products = ExtractProducts(productSegment);
if (products.Count == 0)
{
products.Add(string.Empty);
}
if (versions.Count == products.Count)
{
for (var index = 0; index < versions.Count; index++)
{
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
}
continue;
}
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
{
var groupSize = products.Count / versions.Count;
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
{
var start = versionIndex * groupSize;
var end = start + groupSize;
var version = versions[versionIndex];
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
{
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
}
}
continue;
}
var primaryVersion = versions[0];
foreach (var product in products)
{
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
}
}
if (patches.Count == 0)
{
return Array.Empty<CertCcVendorPatch>();
}
return patches
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
.Distinct(CertCcVendorPatch.Comparer)
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
.ToArray();
}
private static string TrimBulletPrefix(string value)
{
var trimmed = value.TrimStart(BulletPrefixes).Trim();
return trimmed.Length == 0 ? value.Trim() : trimmed;
}
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
{
foreach (var separator in PairSeparators)
{
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
if (parts.Length == 2)
{
productSegment = parts[0];
versionSegment = parts[1];
return true;
}
}
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (whitespaceSplit.Length >= 2)
{
productSegment = string.Join(' ', whitespaceSplit[..^1]);
versionSegment = whitespaceSplit[^1];
return true;
}
productSegment = string.Empty;
versionSegment = string.Empty;
return false;
}
private static List<string> ExtractProducts(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var normalized = segment.Replace('\t', ' ').Trim();
var tokens = normalized
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
.Select(static token => token.Trim())
.Where(static token => token.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
return tokens;
}
private static List<string> ExtractVersions(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var matches = VersionTokenRegex.Matches(segment);
if (matches.Count == 0)
{
return new List<string>();
}
var versions = new List<string>(matches.Count);
foreach (Match match in matches)
{
if (match.Groups.Count == 0)
{
continue;
}
var value = match.Groups[1].Value.Trim();
if (value.Length == 0)
{
continue;
}
versions.Add(value);
}
return versions
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(32)
.ToList();
}
}
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
{
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
{
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
{
if (ReferenceEquals(x, y))
{
return true;
}
if (x is null || y is null)
{
return false;
}
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
}
public int GetHashCode(CertCcVendorPatch obj)
{
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
return HashCode.Combine(product, version);
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace StellaOps.Concelier.Connector.CertCc.Internal;
internal static class CertCcVendorStatementParser
{
private static readonly string[] PairSeparators =
{
"\t",
" - ",
" – ",
" — ",
" : ",
": ",
" :",
":",
};
private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' };
private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' };
// Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a)
private static readonly Regex VersionTokenRegex = new(@"(?<![A-Za-z0-9])(\d+(?:\.\d+){1,3}(?:[A-Za-z0-9\-]+)?)", RegexOptions.Compiled);
public static IReadOnlyList<CertCcVendorPatch> Parse(string? statement)
{
if (string.IsNullOrWhiteSpace(statement))
{
return Array.Empty<CertCcVendorPatch>();
}
var patches = new List<CertCcVendorPatch>();
var lines = statement
.Replace("\r\n", "\n", StringComparison.Ordinal)
.Replace('\r', '\n')
.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (var rawLine in lines)
{
var line = rawLine.Trim();
if (line.Length == 0)
{
continue;
}
line = TrimBulletPrefix(line);
if (line.Length == 0)
{
continue;
}
if (!TrySplitLine(line, out var productSegment, out var versionSegment))
{
continue;
}
var versions = ExtractVersions(versionSegment);
if (versions.Count == 0)
{
continue;
}
var products = ExtractProducts(productSegment);
if (products.Count == 0)
{
products.Add(string.Empty);
}
if (versions.Count == products.Count)
{
for (var index = 0; index < versions.Count; index++)
{
patches.Add(new CertCcVendorPatch(products[index], versions[index], line));
}
continue;
}
if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0)
{
var groupSize = products.Count / versions.Count;
for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++)
{
var start = versionIndex * groupSize;
var end = start + groupSize;
var version = versions[versionIndex];
for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++)
{
patches.Add(new CertCcVendorPatch(products[productIndex], version, line));
}
}
continue;
}
var primaryVersion = versions[0];
foreach (var product in products)
{
patches.Add(new CertCcVendorPatch(product, primaryVersion, line));
}
}
if (patches.Count == 0)
{
return Array.Empty<CertCcVendorPatch>();
}
return patches
.Where(static patch => !string.IsNullOrWhiteSpace(patch.Version))
.Distinct(CertCcVendorPatch.Comparer)
.OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase)
.ThenBy(static patch => patch.Version, StringComparer.Ordinal)
.ToArray();
}
private static string TrimBulletPrefix(string value)
{
var trimmed = value.TrimStart(BulletPrefixes).Trim();
return trimmed.Length == 0 ? value.Trim() : trimmed;
}
private static bool TrySplitLine(string line, out string productSegment, out string versionSegment)
{
foreach (var separator in PairSeparators)
{
var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries);
if (parts.Length == 2)
{
productSegment = parts[0];
versionSegment = parts[1];
return true;
}
}
var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (whitespaceSplit.Length >= 2)
{
productSegment = string.Join(' ', whitespaceSplit[..^1]);
versionSegment = whitespaceSplit[^1];
return true;
}
productSegment = string.Empty;
versionSegment = string.Empty;
return false;
}
private static List<string> ExtractProducts(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var normalized = segment.Replace('\t', ' ').Trim();
var tokens = normalized
.Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries)
.Select(static token => token.Trim())
.Where(static token => token.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
return tokens;
}
private static List<string> ExtractVersions(string segment)
{
if (string.IsNullOrWhiteSpace(segment))
{
return new List<string>();
}
var matches = VersionTokenRegex.Matches(segment);
if (matches.Count == 0)
{
return new List<string>();
}
var versions = new List<string>(matches.Count);
foreach (Match match in matches)
{
if (match.Groups.Count == 0)
{
continue;
}
var value = match.Groups[1].Value.Trim();
if (value.Length == 0)
{
continue;
}
versions.Add(value);
}
return versions
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(32)
.ToList();
}
}
internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine)
{
public static IEqualityComparer<CertCcVendorPatch> Comparer { get; } = new CertCcVendorPatchComparer();
private sealed class CertCcVendorPatchComparer : IEqualityComparer<CertCcVendorPatch>
{
public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y)
{
if (ReferenceEquals(x, y))
{
return true;
}
if (x is null || y is null)
{
return false;
}
return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase);
}
public int GetHashCode(CertCcVendorPatch obj)
{
var product = obj.Product?.ToLowerInvariant() ?? string.Empty;
var version = obj.Version?.ToLowerInvariant() ?? string.Empty;
return HashCode.Combine(product, version);
}
}
}

44
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Jobs.cs
View File

@@ -1,22 +1,22 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertCc;
internal static class CertCcJobKinds
{
public const string Fetch = "source:cert-cc:fetch";
}
internal sealed class CertCcFetchJob : IJob
{
private readonly CertCcConnector _connector;
public CertCcFetchJob(CertCcConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.CertCc;
internal static class CertCcJobKinds
{
public const string Fetch = "source:cert-cc:fetch";
}
internal sealed class CertCcFetchJob : IJob
{
private readonly CertCcConnector _connector;
public CertCcFetchJob(CertCcConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.CertCc/Properties/AssemblyInfo.cs
View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")]