tests fixes and sprints work

This commit is contained in:
master
2026-01-22 19:08:46 +02:00
parent c32fff8f86
commit 726d70dc7f
881 changed files with 134434 additions and 6228 deletions

View File

@@ -0,0 +1,42 @@
# Platform Analytics Ingestion (StellaOps.Platform.Analytics)
## Mission
- Ingest SBOM, vulnerability, and attestation events into the analytics schema.
- Normalize and store raw payloads for replayable audits.
- Provide deterministic, tenant-scoped analytics data for downstream queries.
## Roles
- Backend engineer: ingestion services, normalization, persistence, idempotency.
- QA automation engineer: deterministic fixtures and schema validation tests.
- Docs maintainer: ingestion contracts, data flow, and runbooks.
## Operating principles
- Idempotent upserts; safe replay of the same input.
- Deterministic ordering and UTC timestamps.
- Offline-first: no hidden network calls; rely on local feeds.
- Tenancy-aware: enforce tenant context on every ingest.
- Auditability: store raw payloads and ingestion metadata.
## Working directory
- `src/Platform/StellaOps.Platform.Analytics/`
## Testing expectations
- Unit tests for normalization, deduplication, and contract parsing.
- Integration tests using deterministic fixtures; avoid network.
- Validate materialized view refresh outputs with frozen datasets.
## Working agreements
- Update sprint status in `docs/implplan/SPRINT_*.md` and local `TASKS.md` if added.
- Record contract changes in sprint Decisions & Risks with doc links.
- Keep ingestion schemas aligned with `docs/db/analytics_schema.sql`.
## Required reading
- `docs/modules/analytics/README.md`
- `docs/modules/analytics/architecture.md`
- `docs/modules/analytics/queries.md`
- `docs/modules/scanner/architecture.md`
- `docs/modules/concelier/architecture.md`
- `docs/modules/excititor/architecture.md`
- `docs/modules/attestor/architecture.md`
- `docs/sboms/DETERMINISM.md`
- `src/Platform/AGENTS.md`

View File

@@ -0,0 +1,56 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Platform.Analytics.Models;
public sealed record AdvisoryObservationUpdatedEvent
{
[JsonPropertyName("eventId")]
public string EventId { get; init; } = string.Empty;
[JsonPropertyName("tenantId")]
public string TenantId { get; init; } = string.Empty;
[JsonPropertyName("advisoryId")]
public string AdvisoryId { get; init; } = string.Empty;
[JsonPropertyName("linksetSummary")]
public AdvisoryLinksetSummary LinksetSummary { get; init; } = new();
[JsonPropertyName("documentSha")]
public string DocumentSha { get; init; } = string.Empty;
[JsonPropertyName("replayCursor")]
public string ReplayCursor { get; init; } = string.Empty;
}
public sealed record AdvisoryLinksetSummary
{
[JsonPropertyName("purls")]
public IReadOnlyList<string> Purls { get; init; } = Array.Empty<string>();
}
public sealed record AdvisoryLinksetUpdatedEvent
{
[JsonPropertyName("eventId")]
public string EventId { get; init; } = string.Empty;
[JsonPropertyName("tenantId")]
public string TenantId { get; init; } = string.Empty;
[JsonPropertyName("linksetId")]
public string LinksetId { get; init; } = string.Empty;
[JsonPropertyName("advisoryId")]
public string AdvisoryId { get; init; } = string.Empty;
[JsonPropertyName("source")]
public string Source { get; init; } = string.Empty;
[JsonPropertyName("createdAt")]
public DateTimeOffset CreatedAt { get; init; }
[JsonPropertyName("replayCursor")]
public string ReplayCursor { get; init; } = string.Empty;
}

View File

@@ -0,0 +1,79 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Platform.Analytics.Models;
public sealed record RekorEntryEvent
{
[JsonPropertyName("eventId")]
public string EventId { get; init; } = string.Empty;
[JsonPropertyName("eventType")]
public string EventType { get; init; } = string.Empty;
[JsonPropertyName("tenant")]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("bundleDigest")]
public string BundleDigest { get; init; } = string.Empty;
[JsonPropertyName("predicateType")]
public string PredicateType { get; init; } = string.Empty;
[JsonPropertyName("logIndex")]
public long LogIndex { get; init; }
[JsonPropertyName("logId")]
public string LogId { get; init; } = string.Empty;
[JsonPropertyName("entryUuid")]
public string EntryUuid { get; init; } = string.Empty;
[JsonPropertyName("integratedTime")]
public long IntegratedTime { get; init; }
[JsonPropertyName("integratedTimeRfc3339")]
public string IntegratedTimeRfc3339 { get; init; } = string.Empty;
[JsonPropertyName("entryUrl")]
public string? EntryUrl { get; init; }
[JsonPropertyName("inclusionVerified")]
public bool InclusionVerified { get; init; }
[JsonPropertyName("reanalysisHints")]
public RekorReanalysisHints? ReanalysisHints { get; init; }
[JsonPropertyName("createdAtUtc")]
public DateTimeOffset CreatedAtUtc { get; init; }
[JsonPropertyName("traceId")]
public string? TraceId { get; init; }
}
public sealed record RekorReanalysisHints
{
[JsonPropertyName("cveIds")]
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
[JsonPropertyName("productKeys")]
public IReadOnlyList<string> ProductKeys { get; init; } = Array.Empty<string>();
[JsonPropertyName("artifactDigests")]
public IReadOnlyList<string> ArtifactDigests { get; init; } = Array.Empty<string>();
[JsonPropertyName("mayAffectDecision")]
public bool MayAffectDecision { get; init; }
[JsonPropertyName("reanalysisScope")]
public string ReanalysisScope { get; init; } = "none";
}
public static class RekorEventTypes
{
public const string EntryLogged = "rekor.entry.logged";
public const string EntryQueued = "rekor.entry.queued";
public const string InclusionVerified = "rekor.inclusion.verified";
public const string EntryFailed = "rekor.entry.failed";
}

View File

@@ -0,0 +1,141 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Scanner.Surface.FS;
namespace StellaOps.Platform.Analytics.Models;
public sealed record OrchestratorEventEnvelope
{
[JsonPropertyName("eventId")]
public Guid EventId { get; init; }
[JsonPropertyName("kind")]
public string Kind { get; init; } = string.Empty;
[JsonPropertyName("version")]
public int Version { get; init; } = 1;
[JsonPropertyName("tenant")]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("occurredAt")]
public DateTimeOffset OccurredAt { get; init; }
[JsonPropertyName("recordedAt")]
public DateTimeOffset? RecordedAt { get; init; }
[JsonPropertyName("source")]
public string? Source { get; init; }
[JsonPropertyName("idempotencyKey")]
public string? IdempotencyKey { get; init; }
[JsonPropertyName("correlationId")]
public string? CorrelationId { get; init; }
[JsonPropertyName("traceId")]
public string? TraceId { get; init; }
[JsonPropertyName("scope")]
public OrchestratorEventScope? Scope { get; init; }
[JsonPropertyName("payload")]
public JsonElement? Payload { get; init; }
}
public sealed record OrchestratorEventScope
{
[JsonPropertyName("namespace")]
public string? Namespace { get; init; }
[JsonPropertyName("repo")]
public string? Repo { get; init; }
[JsonPropertyName("digest")]
public string? Digest { get; init; }
[JsonPropertyName("component")]
public string? Component { get; init; }
[JsonPropertyName("image")]
public string? Image { get; init; }
}
public sealed record ReportReadyEventPayload
{
[JsonPropertyName("reportId")]
public string ReportId { get; init; } = string.Empty;
[JsonPropertyName("scanId")]
public string? ScanId { get; init; }
[JsonPropertyName("imageDigest")]
public string ImageDigest { get; init; } = string.Empty;
[JsonPropertyName("generatedAt")]
public DateTimeOffset GeneratedAt { get; init; }
[JsonPropertyName("summary")]
public ReportSummaryPayload Summary { get; init; } = new();
[JsonPropertyName("report")]
public ReportDocumentPayload Report { get; init; } = new();
}
public sealed record ReportSummaryPayload
{
[JsonPropertyName("total")]
public int Total { get; init; }
[JsonPropertyName("blocked")]
public int Blocked { get; init; }
[JsonPropertyName("warned")]
public int Warned { get; init; }
[JsonPropertyName("ignored")]
public int Ignored { get; init; }
[JsonPropertyName("quieted")]
public int Quieted { get; init; }
}
public sealed record ReportDocumentPayload
{
[JsonPropertyName("reportId")]
public string ReportId { get; init; } = string.Empty;
[JsonPropertyName("imageDigest")]
public string ImageDigest { get; init; } = string.Empty;
[JsonPropertyName("generatedAt")]
public DateTimeOffset GeneratedAt { get; init; }
[JsonPropertyName("surface")]
public SurfacePointersPayload? Surface { get; init; }
}
public sealed record SurfacePointersPayload
{
[JsonPropertyName("tenant")]
public string Tenant { get; init; } = string.Empty;
[JsonPropertyName("generatedAt")]
public DateTimeOffset GeneratedAt { get; init; }
[JsonPropertyName("manifestDigest")]
public string ManifestDigest { get; init; } = string.Empty;
[JsonPropertyName("manifestUri")]
public string? ManifestUri { get; init; }
[JsonPropertyName("manifest")]
public SurfaceManifestDocument Manifest { get; init; } = new();
}
public static class OrchestratorEventKinds
{
public const string ScannerReportReady = "scanner.event.report.ready";
}

View File

@@ -0,0 +1,93 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Platform.Analytics.Options;
public sealed class AnalyticsIngestionOptions
{
public const string SectionName = "Platform:AnalyticsIngestion";
public bool Enabled { get; set; } = true;
public string? PostgresConnectionString { get; set; }
public string SchemaVersion { get; set; } = "1.0.0";
public string IngestVersion { get; set; } = "1.0.0";
public AnalyticsStreamOptions Streams { get; set; } = new();
public AnalyticsCasOptions Cas { get; set; } = new();
public AnalyticsAttestationOptions Attestations { get; set; } = new();
public List<string> AllowedTenants { get; set; } = new();
public void Normalize()
{
SchemaVersion = SchemaVersion?.Trim() ?? "1.0.0";
IngestVersion = IngestVersion?.Trim() ?? "1.0.0";
PostgresConnectionString = string.IsNullOrWhiteSpace(PostgresConnectionString)
? null
: PostgresConnectionString.Trim();
Streams ??= new AnalyticsStreamOptions();
Cas ??= new AnalyticsCasOptions();
Attestations ??= new AnalyticsAttestationOptions();
AllowedTenants ??= new List<string>();
Streams.Normalize();
Cas.Normalize();
Attestations.Normalize();
AllowedTenants = AllowedTenants
.Where(value => !string.IsNullOrWhiteSpace(value))
.Select(value => value.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
}
public void Validate()
{
if (Enabled && string.IsNullOrWhiteSpace(PostgresConnectionString))
{
throw new InvalidOperationException(
"Analytics ingestion requires a Postgres connection string.");
}
}
}
public sealed class AnalyticsStreamOptions
{
public string ScannerStream { get; set; } = "orchestrator:events";
public string ConcelierObservationStream { get; set; } = "concelier:advisory.observation.updated:v1";
public string ConcelierLinksetStream { get; set; } = "concelier:advisory.linkset.updated:v1";
public string AttestorStream { get; set; } = "attestor:events";
public bool StartFromBeginning { get; set; } = false;
public void Normalize()
{
ScannerStream = NormalizeName(ScannerStream);
ConcelierObservationStream = NormalizeName(ConcelierObservationStream);
ConcelierLinksetStream = NormalizeName(ConcelierLinksetStream);
AttestorStream = NormalizeName(AttestorStream);
}
private static string NormalizeName(string value)
=> string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim();
}
public sealed class AnalyticsCasOptions
{
public string? RootPath { get; set; }
public string? DefaultBucket { get; set; }
public void Normalize()
{
RootPath = string.IsNullOrWhiteSpace(RootPath) ? null : RootPath.Trim();
DefaultBucket = string.IsNullOrWhiteSpace(DefaultBucket) ? null : DefaultBucket.Trim();
}
}
public sealed class AnalyticsAttestationOptions
{
public string BundleUriTemplate { get; set; } = "bundle:{digest}";
public void Normalize()
{
BundleUriTemplate = string.IsNullOrWhiteSpace(BundleUriTemplate)
? "bundle:{digest}"
: BundleUriTemplate.Trim();
}
}

View File

@@ -0,0 +1,59 @@
// SPDX-License-Identifier: BUSL-1.1
// Copyright (c) 2026 stella-ops.org
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Concelier.SbomIntegration.Parsing;
using StellaOps.Platform.Analytics.Options;
using StellaOps.Platform.Analytics.Services;
namespace StellaOps.Platform.Analytics;
public static class ServiceCollectionExtensions
{
/// <summary>
/// Registers analytics ingestion services (SBOM, vulnerability correlation, attestation).
/// </summary>
/// <param name="services">The service collection.</param>
/// <param name="configuration">The configuration root.</param>
/// <returns>The service collection for chaining.</returns>
public static IServiceCollection AddAnalyticsIngestion(
this IServiceCollection services,
IConfiguration configuration,
string? defaultConnectionString = null)
{
// Bind options
services.AddOptions<AnalyticsIngestionOptions>()
.Bind(configuration.GetSection(AnalyticsIngestionOptions.SectionName))
.PostConfigure(options =>
{
if (string.IsNullOrWhiteSpace(options.PostgresConnectionString) &&
!string.IsNullOrWhiteSpace(defaultConnectionString))
{
options.PostgresConnectionString = defaultConnectionString;
}
options.Normalize();
})
.ValidateOnStart();
// Data source and CAS reader
services.AddSingleton<AnalyticsIngestionDataSource>();
services.AddSingleton<ICasContentReader, FileCasContentReader>();
// SBOM parser (from Concelier.SbomIntegration)
services.AddSingleton<IParsedSbomParser, ParsedSbomParser>();
// Vulnerability correlation service (also a BackgroundService)
services.AddSingleton<IVulnerabilityCorrelationService, VulnerabilityCorrelationService>();
services.AddHostedService(sp => (VulnerabilityCorrelationService)sp.GetRequiredService<IVulnerabilityCorrelationService>());
// SBOM ingestion service
services.AddHostedService<AnalyticsIngestionService>();
// Attestation ingestion service
services.AddHostedService<AttestationIngestionService>();
return services;
}
}

View File

@@ -0,0 +1,64 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Npgsql;
using StellaOps.Platform.Analytics.Options;
namespace StellaOps.Platform.Analytics.Services;
public sealed class AnalyticsIngestionDataSource : IAsyncDisposable
{
private readonly ILogger<AnalyticsIngestionDataSource> _logger;
private readonly string? _connectionString;
private NpgsqlDataSource? _dataSource;
public AnalyticsIngestionDataSource(
IOptions<AnalyticsIngestionOptions> options,
ILogger<AnalyticsIngestionDataSource> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_connectionString = options?.Value.PostgresConnectionString;
}
public bool IsConfigured => !string.IsNullOrWhiteSpace(_connectionString);
public async Task<NpgsqlConnection?> OpenConnectionAsync(CancellationToken cancellationToken)
{
if (!IsConfigured)
{
return null;
}
_dataSource ??= new NpgsqlDataSourceBuilder(_connectionString!)
{
Name = "StellaOps.Platform.Analytics.Ingestion"
}.Build();
var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
await ConfigureSessionAsync(connection, cancellationToken).ConfigureAwait(false);
return connection;
}
public async ValueTask DisposeAsync()
{
if (_dataSource is null)
{
return;
}
await _dataSource.DisposeAsync().ConfigureAwait(false);
}
private async Task ConfigureSessionAsync(NpgsqlConnection connection, CancellationToken cancellationToken)
{
await using var tzCommand = new NpgsqlCommand("SET TIME ZONE 'UTC';", connection);
await tzCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
await using var schemaCommand = new NpgsqlCommand("SET search_path TO analytics, public;", connection);
await schemaCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Configured analytics ingestion session for PostgreSQL connection.");
}
}

View File

@@ -0,0 +1,877 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Npgsql;
using NpgsqlTypes;
using StellaOps.Concelier.SbomIntegration.Models;
using StellaOps.Concelier.SbomIntegration.Parsing;
using StellaOps.Messaging;
using StellaOps.Messaging.Abstractions;
using StellaOps.Platform.Analytics.Models;
using StellaOps.Platform.Analytics.Options;
using StellaOps.Platform.Analytics.Utilities;
using StellaOps.Scanner.Surface.FS;
namespace StellaOps.Platform.Analytics.Services;
public sealed class AnalyticsIngestionService : BackgroundService
{
private readonly AnalyticsIngestionOptions _options;
private readonly AnalyticsIngestionDataSource _dataSource;
private readonly ICasContentReader _casReader;
private readonly IParsedSbomParser _sbomParser;
private readonly IVulnerabilityCorrelationService? _correlationService;
private readonly ILogger<AnalyticsIngestionService> _logger;
private readonly IEventStream<OrchestratorEventEnvelope>? _eventStream;
private readonly JsonSerializerOptions _jsonOptions = new()
{
PropertyNameCaseInsensitive = true
};
public AnalyticsIngestionService(
IOptions<AnalyticsIngestionOptions> options,
AnalyticsIngestionDataSource dataSource,
ICasContentReader casReader,
IParsedSbomParser sbomParser,
ILogger<AnalyticsIngestionService> logger,
IEventStreamFactory? eventStreamFactory = null,
IVulnerabilityCorrelationService? correlationService = null)
{
_options = options?.Value ?? new AnalyticsIngestionOptions();
_options.Normalize();
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
_casReader = casReader ?? throw new ArgumentNullException(nameof(casReader));
_sbomParser = sbomParser ?? throw new ArgumentNullException(nameof(sbomParser));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_correlationService = correlationService;
if (eventStreamFactory is not null && !string.IsNullOrWhiteSpace(_options.Streams.ScannerStream))
{
_eventStream = eventStreamFactory.Create<OrchestratorEventEnvelope>(new EventStreamOptions
{
StreamName = _options.Streams.ScannerStream
});
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (!_options.Enabled)
{
_logger.LogInformation("Analytics ingestion disabled by configuration.");
return;
}
if (_eventStream is null)
{
_logger.LogWarning("Analytics ingestion disabled: no event stream configured.");
return;
}
var position = _options.Streams.StartFromBeginning
? StreamPosition.Beginning
: StreamPosition.End;
_logger.LogInformation(
"Analytics ingestion started; subscribing to {StreamName} from {Position}.",
_eventStream.StreamName,
position.Value);
try
{
await foreach (var streamEvent in _eventStream.SubscribeAsync(position, stoppingToken))
{
await HandleEventAsync(streamEvent.Event, stoppingToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
_logger.LogInformation("Analytics ingestion stopped.");
}
catch (Exception ex)
{
_logger.LogError(ex, "Analytics ingestion failed.");
throw;
}
}
private async Task HandleEventAsync(OrchestratorEventEnvelope envelope, CancellationToken cancellationToken)
{
if (!string.Equals(envelope.Kind, OrchestratorEventKinds.ScannerReportReady, StringComparison.OrdinalIgnoreCase))
{
return;
}
if (!IsTenantAllowed(envelope.Tenant))
{
_logger.LogDebug("Skipping scanner event {EventId}; tenant {Tenant} not allowed.", envelope.EventId, envelope.Tenant);
return;
}
if (envelope.Payload is null || envelope.Payload.Value.ValueKind == JsonValueKind.Undefined)
{
_logger.LogWarning("Scanner report event {EventId} missing payload.", envelope.EventId);
return;
}
ReportReadyEventPayload? payload;
try
{
payload = envelope.Payload.Value.Deserialize<ReportReadyEventPayload>(_jsonOptions);
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Failed to parse scanner report payload for event {EventId}.", envelope.EventId);
return;
}
if (payload is null)
{
_logger.LogWarning("Scanner report payload empty for event {EventId}.", envelope.EventId);
return;
}
await IngestSbomAsync(envelope, payload, cancellationToken).ConfigureAwait(false);
}
private async Task IngestSbomAsync(
OrchestratorEventEnvelope envelope,
ReportReadyEventPayload payload,
CancellationToken cancellationToken)
{
var surface = payload.Report.Surface;
var manifest = await ResolveManifestAsync(surface, cancellationToken).ConfigureAwait(false);
if (manifest is null)
{
_logger.LogWarning("Scanner report {ReportId} missing surface manifest.", payload.ReportId);
return;
}
var sbomArtifact = SelectSbomArtifact(manifest.Artifacts);
if (sbomArtifact is null)
{
_logger.LogWarning("Scanner report {ReportId} contains no SBOM artifacts.", payload.ReportId);
return;
}
var sbomContent = await ReadContentAsync(sbomArtifact.Uri, cancellationToken).ConfigureAwait(false);
if (sbomContent is null)
{
_logger.LogWarning("Failed to read SBOM content for report {ReportId}.", payload.ReportId);
return;
}
var sbomFormat = ResolveSbomFormat(sbomArtifact);
ParsedSbom parsedSbom;
await using (var sbomStream = new MemoryStream(sbomContent.Bytes, writable: false))
{
parsedSbom = await _sbomParser.ParseAsync(sbomStream, sbomFormat, cancellationToken)
.ConfigureAwait(false);
}
var artifactDigest = NormalizeDigest(payload.ImageDigest)
?? NormalizeDigest(envelope.Scope?.Digest);
if (string.IsNullOrWhiteSpace(artifactDigest))
{
_logger.LogWarning("Scanner report {ReportId} missing artifact digest.", payload.ReportId);
return;
}
var artifactName = ResolveArtifactName(envelope);
var artifactVersion = ResolveArtifactVersion(envelope);
var sbomDigest = NormalizeDigest(sbomArtifact.Digest) ?? sbomContent.Digest;
var storageUri = sbomArtifact.Uri;
var contentSize = sbomArtifact.SizeBytes > 0 ? sbomArtifact.SizeBytes : sbomContent.Length;
var formatLabel = NormalizeSbomFormat(parsedSbom.Format, sbomFormat);
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
if (connection is null)
{
_logger.LogWarning("Analytics ingestion skipped: database is not configured.");
return;
}
var componentSeeds = BuildComponentSeeds(parsedSbom);
var componentCount = componentSeeds.Count;
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
var artifactId = await UpsertArtifactAsync(
connection,
transaction,
artifactDigest,
artifactName,
artifactVersion,
sbomDigest,
formatLabel,
parsedSbom.SpecVersion,
componentCount,
cancellationToken).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(sbomDigest))
{
await UpsertRawSbomAsync(
connection,
transaction,
artifactId,
sbomDigest,
contentSize,
storageUri,
formatLabel,
parsedSbom.SpecVersion,
cancellationToken).ConfigureAwait(false);
}
var componentIds = new Dictionary<ComponentKey, Guid>();
foreach (var seed in componentSeeds)
{
var key = new ComponentKey(seed.Purl, seed.HashSha256);
if (!componentIds.TryGetValue(key, out var componentId))
{
componentId = await UpsertComponentAsync(
connection,
transaction,
seed,
cancellationToken).ConfigureAwait(false);
componentIds[key] = componentId;
}
var inserted = await InsertArtifactComponentAsync(
connection,
transaction,
artifactId,
componentId,
seed,
cancellationToken).ConfigureAwait(false);
if (inserted)
{
await IncrementComponentCountsAsync(
connection,
transaction,
componentId,
cancellationToken).ConfigureAwait(false);
}
}
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
if (_correlationService is not null)
{
var purls = componentSeeds
.Select(seed => seed.Purl)
.Where(purl => !string.IsNullOrWhiteSpace(purl))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
await _correlationService.CorrelateForPurlsAsync(purls, cancellationToken)
.ConfigureAwait(false);
await _correlationService.UpdateArtifactCountsAsync(artifactId, cancellationToken)
.ConfigureAwait(false);
}
}
private async Task<SurfaceManifestDocument?> ResolveManifestAsync(
SurfacePointersPayload? surface,
CancellationToken cancellationToken)
{
if (surface is null)
{
return null;
}
if (surface.Manifest.Artifacts.Count > 0)
{
return surface.Manifest;
}
if (string.IsNullOrWhiteSpace(surface.ManifestUri))
{
return null;
}
var manifestContent = await ReadContentAsync(surface.ManifestUri, cancellationToken).ConfigureAwait(false);
if (manifestContent is null)
{
return null;
}
try
{
return JsonSerializer.Deserialize<SurfaceManifestDocument>(
manifestContent.Bytes,
_jsonOptions);
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Failed to deserialize surface manifest from {ManifestUri}.", surface.ManifestUri);
return null;
}
}
internal static SurfaceManifestArtifact? SelectSbomArtifact(IReadOnlyList<SurfaceManifestArtifact> artifacts)
{
if (artifacts.Count == 0)
{
return null;
}
SurfaceManifestArtifact? Find(Func<SurfaceManifestArtifact, bool> predicate)
=> artifacts.FirstOrDefault(predicate);
return Find(a => string.Equals(a.Kind, "sbom-inventory", StringComparison.OrdinalIgnoreCase))
?? Find(a => string.Equals(a.View, "inventory", StringComparison.OrdinalIgnoreCase))
?? Find(a => string.Equals(a.Kind, "sbom-usage", StringComparison.OrdinalIgnoreCase))
?? Find(a => string.Equals(a.View, "usage", StringComparison.OrdinalIgnoreCase))
?? Find(a => a.Kind.Contains("sbom", StringComparison.OrdinalIgnoreCase))
?? Find(a => a.MediaType.Contains("cyclonedx", StringComparison.OrdinalIgnoreCase))
?? Find(a => a.MediaType.Contains("spdx", StringComparison.OrdinalIgnoreCase));
}
private async Task<ContentPayload?> ReadContentAsync(string uri, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(uri))
{
return null;
}
var casContent = await _casReader.OpenReadAsync(uri, cancellationToken).ConfigureAwait(false);
if (casContent is null)
{
return null;
}
await using var stream = casContent.Stream;
using var buffer = new MemoryStream();
await stream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
var bytes = buffer.ToArray();
var digest = Sha256Hasher.Compute(bytes);
return new ContentPayload(bytes, casContent.Length ?? bytes.Length, digest);
}
internal static SbomFormat ResolveSbomFormat(SurfaceManifestArtifact artifact)
{
var format = artifact.Format?.ToLowerInvariant() ?? string.Empty;
if (format.Contains("spdx", StringComparison.OrdinalIgnoreCase))
{
return SbomFormat.SPDX;
}
if (format.Contains("cdx", StringComparison.OrdinalIgnoreCase) ||
format.Contains("cyclonedx", StringComparison.OrdinalIgnoreCase))
{
return SbomFormat.CycloneDX;
}
var media = artifact.MediaType?.ToLowerInvariant() ?? string.Empty;
return media.Contains("spdx", StringComparison.OrdinalIgnoreCase)
? SbomFormat.SPDX
: SbomFormat.CycloneDX;
}
internal static string NormalizeSbomFormat(string parsedFormat, SbomFormat fallback)
{
if (parsedFormat.Equals("spdx", StringComparison.OrdinalIgnoreCase))
{
return "spdx";
}
if (parsedFormat.Equals("cyclonedx", StringComparison.OrdinalIgnoreCase))
{
return "cyclonedx";
}
return fallback == SbomFormat.SPDX ? "spdx" : "cyclonedx";
}
internal static string NormalizeDigest(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return string.Empty;
}
var trimmed = digest.Trim();
return trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
? $"sha256:{trimmed[7..].ToLowerInvariant()}"
: $"sha256:{trimmed.ToLowerInvariant()}";
}
internal static string ResolveArtifactName(OrchestratorEventEnvelope envelope)
{
if (!string.IsNullOrWhiteSpace(envelope.Scope?.Repo))
{
return envelope.Scope!.Repo!;
}
return envelope.Scope?.Image ?? envelope.Scope?.Component ?? "unknown";
}
internal static string? ResolveArtifactVersion(OrchestratorEventEnvelope envelope)
{
if (string.IsNullOrWhiteSpace(envelope.Scope?.Image))
{
return null;
}
var image = envelope.Scope.Image;
var tagIndex = image.LastIndexOf(':');
if (tagIndex > 0 && tagIndex < image.Length - 1)
{
return image[(tagIndex + 1)..];
}
return null;
}
private List<ComponentSeed> BuildComponentSeeds(ParsedSbom sbom)
{
var dependencyMap = BuildDependencyMap(sbom);
var paths = BuildDependencyPaths(sbom, dependencyMap);
var seeds = new List<ComponentSeed>();
foreach (var component in sbom.Components)
{
var purl = !string.IsNullOrWhiteSpace(component.Purl)
? PurlParser.Parse(component.Purl).Normalized
: PurlParser.BuildGeneric(component.Name, component.Version);
var hash = ResolveComponentHash(component, purl);
var licenseExpression = LicenseExpressionRenderer.BuildExpression(component.Licenses);
var supplier = component.Supplier?.Name ?? component.Publisher ?? sbom.Metadata.Supplier ?? sbom.Metadata.Manufacturer;
paths.TryGetValue(component.BomRef, out var dependencyPath);
var depth = dependencyPath?.Length > 0 ? dependencyPath.Length - 1 : 0;
var introducedVia = dependencyPath is { Length: > 1 } ? dependencyPath[^2] : null;
seeds.Add(new ComponentSeed(
component.BomRef,
purl,
hash,
component.Name,
component.Version,
MapComponentType(component.Type),
supplier,
licenseExpression,
licenseExpression,
component.Description,
component.Cpe,
MapScope(component.Scope),
dependencyPath,
depth,
introducedVia));
}
return seeds
.GroupBy(seed => new ComponentKey(seed.Purl, seed.HashSha256))
.Select(group => group
.OrderBy(seed => seed.Depth)
.ThenBy(seed => seed.BomRef, StringComparer.Ordinal)
.First())
.ToList();
}
internal static Dictionary<string, IReadOnlyList<string>> BuildDependencyMap(ParsedSbom sbom)
{
var map = new Dictionary<string, IReadOnlyList<string>>(StringComparer.Ordinal);
foreach (var dependency in sbom.Dependencies)
{
if (string.IsNullOrWhiteSpace(dependency.SourceRef))
{
continue;
}
var list = dependency.DependsOn
.Where(value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.Ordinal)
.OrderBy(value => value, StringComparer.Ordinal)
.ToArray();
if (list.Length > 0)
{
map[dependency.SourceRef] = list;
}
}
return map;
}
internal static Dictionary<string, string[]> BuildDependencyPaths(
ParsedSbom sbom,
Dictionary<string, IReadOnlyList<string>> dependencyMap)
{
var paths = new Dictionary<string, string[]>(StringComparer.Ordinal);
var root = sbom.Metadata.RootComponentRef;
if (string.IsNullOrWhiteSpace(root))
{
return paths;
}
var queue = new Queue<string>();
paths[root] = new[] { root };
queue.Enqueue(root);
while (queue.Count > 0)
{
var current = queue.Dequeue();
if (!dependencyMap.TryGetValue(current, out var children))
{
continue;
}
foreach (var child in children)
{
if (paths.ContainsKey(child))
{
continue;
}
var parentPath = paths[current];
var childPath = new string[parentPath.Length + 1];
Array.Copy(parentPath, childPath, parentPath.Length);
childPath[^1] = child;
paths[child] = childPath;
queue.Enqueue(child);
}
}
return paths;
}
internal static string ResolveComponentHash(ParsedComponent component, string purl)
{
var hash = component.Hashes
.FirstOrDefault(h => h.Algorithm.Equals("sha-256", StringComparison.OrdinalIgnoreCase)
|| h.Algorithm.Equals("sha256", StringComparison.OrdinalIgnoreCase))
?.Value;
return !string.IsNullOrWhiteSpace(hash) ? NormalizeDigest(hash) : Sha256Hasher.Compute(purl);
}
internal static string MapComponentType(string? type)
{
if (string.IsNullOrWhiteSpace(type))
{
return "library";
}
var normalized = type.Trim().ToLowerInvariant();
return normalized switch
{
"application" => "application",
"container" => "container",
"framework" => "framework",
"operating-system" => "operating-system",
"operating system" => "operating-system",
"os" => "operating-system",
"device" => "device",
"firmware" => "firmware",
"file" => "file",
_ => "library"
};
}
internal static string MapScope(ComponentScope scope)
{
return scope switch
{
ComponentScope.Optional => "optional",
ComponentScope.Excluded => "excluded",
ComponentScope.Unknown => "unknown",
_ => "required"
};
}
private async Task<Guid> UpsertArtifactAsync(
NpgsqlConnection connection,
NpgsqlTransaction transaction,
string digest,
string name,
string? version,
string sbomDigest,
string sbomFormat,
string sbomSpecVersion,
int componentCount,
CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO analytics.artifacts (
artifact_type,
name,
version,
digest,
sbom_digest,
sbom_format,
sbom_spec_version,
component_count,
vulnerability_count,
critical_count,
high_count,
medium_count,
low_count,
updated_at
)
VALUES (
@artifact_type,
@name,
@version,
@digest,
@sbom_digest,
@sbom_format,
@sbom_spec_version,
@component_count,
0,
0,
0,
0,
0,
now()
)
ON CONFLICT (digest) DO UPDATE SET
name = EXCLUDED.name,
version = COALESCE(EXCLUDED.version, analytics.artifacts.version),
sbom_digest = EXCLUDED.sbom_digest,
sbom_format = EXCLUDED.sbom_format,
sbom_spec_version = EXCLUDED.sbom_spec_version,
component_count = EXCLUDED.component_count,
updated_at = now()
RETURNING artifact_id;
""";
await using var command = new NpgsqlCommand(sql, connection, transaction);
command.Parameters.AddWithValue("artifact_type", "container");
command.Parameters.AddWithValue("name", name);
command.Parameters.AddWithValue("version", (object?)version ?? DBNull.Value);
command.Parameters.AddWithValue("digest", digest);
command.Parameters.AddWithValue("sbom_digest", sbomDigest);
command.Parameters.AddWithValue("sbom_format", sbomFormat);
command.Parameters.AddWithValue("sbom_spec_version", sbomSpecVersion);
command.Parameters.AddWithValue("component_count", componentCount);
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return result is Guid id ? id : Guid.Empty;
}
private async Task UpsertRawSbomAsync(
NpgsqlConnection connection,
NpgsqlTransaction transaction,
Guid artifactId,
string contentHash,
long contentSize,
string storageUri,
string format,
string specVersion,
CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO analytics.raw_sboms (
artifact_id,
format,
spec_version,
content_hash,
content_size,
storage_uri,
ingest_version,
schema_version
)
VALUES (
@artifact_id,
@format,
@spec_version,
@content_hash,
@content_size,
@storage_uri,
@ingest_version,
@schema_version
)
ON CONFLICT (content_hash) DO NOTHING;
""";
await using var command = new NpgsqlCommand(sql, connection, transaction);
command.Parameters.AddWithValue("artifact_id", artifactId);
command.Parameters.AddWithValue("format", format);
command.Parameters.AddWithValue("spec_version", specVersion);
command.Parameters.AddWithValue("content_hash", contentHash);
command.Parameters.AddWithValue("content_size", contentSize);
command.Parameters.AddWithValue("storage_uri", storageUri);
command.Parameters.AddWithValue("ingest_version", _options.IngestVersion);
command.Parameters.AddWithValue("schema_version", _options.SchemaVersion);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private async Task<Guid> UpsertComponentAsync(
NpgsqlConnection connection,
NpgsqlTransaction transaction,
ComponentSeed seed,
CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO analytics.components (
purl,
purl_type,
purl_namespace,
purl_name,
purl_version,
hash_sha256,
name,
version,
description,
component_type,
supplier,
supplier_normalized,
license_declared,
license_concluded,
license_category,
cpe
)
SELECT
@purl,
parsed.purl_type,
parsed.purl_namespace,
parsed.purl_name,
parsed.purl_version,
@hash_sha256,
@name,
@version,
@description,
@component_type,
@supplier,
analytics.normalize_supplier(@supplier),
@license_declared,
@license_concluded,
analytics.categorize_license(@license_concluded),
@cpe
FROM analytics.parse_purl(@purl) AS parsed
ON CONFLICT (purl, hash_sha256) DO UPDATE SET
last_seen_at = now(),
updated_at = now(),
supplier = COALESCE(EXCLUDED.supplier, analytics.components.supplier),
supplier_normalized = COALESCE(EXCLUDED.supplier_normalized, analytics.components.supplier_normalized),
license_declared = COALESCE(EXCLUDED.license_declared, analytics.components.license_declared),
license_concluded = COALESCE(EXCLUDED.license_concluded, analytics.components.license_concluded),
license_category = COALESCE(EXCLUDED.license_category, analytics.components.license_category),
description = COALESCE(EXCLUDED.description, analytics.components.description),
cpe = COALESCE(EXCLUDED.cpe, analytics.components.cpe),
component_type = COALESCE(EXCLUDED.component_type, analytics.components.component_type),
name = COALESCE(EXCLUDED.name, analytics.components.name),
version = COALESCE(EXCLUDED.version, analytics.components.version)
RETURNING component_id;
""";
await using var command = new NpgsqlCommand(sql, connection, transaction);
command.Parameters.AddWithValue("purl", seed.Purl);
command.Parameters.AddWithValue("hash_sha256", seed.HashSha256);
command.Parameters.AddWithValue("name", seed.Name);
command.Parameters.AddWithValue("version", (object?)seed.Version ?? DBNull.Value);
command.Parameters.AddWithValue("description", (object?)seed.Description ?? DBNull.Value);
command.Parameters.AddWithValue("component_type", seed.ComponentType);
command.Parameters.AddWithValue("supplier", (object?)seed.Supplier ?? DBNull.Value);
command.Parameters.AddWithValue("license_declared", (object?)seed.LicenseDeclared ?? DBNull.Value);
command.Parameters.AddWithValue("license_concluded", (object?)seed.LicenseConcluded ?? DBNull.Value);
command.Parameters.AddWithValue("cpe", (object?)seed.Cpe ?? DBNull.Value);
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return result is Guid id ? id : Guid.Empty;
}
private async Task<bool> InsertArtifactComponentAsync(
NpgsqlConnection connection,
NpgsqlTransaction transaction,
Guid artifactId,
Guid componentId,
ComponentSeed seed,
CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO analytics.artifact_components (
artifact_id,
component_id,
bom_ref,
scope,
dependency_path,
depth,
introduced_via
)
VALUES (
@artifact_id,
@component_id,
@bom_ref,
@scope,
@dependency_path,
@depth,
@introduced_via
)
ON CONFLICT (artifact_id, component_id) DO NOTHING;
""";
await using var command = new NpgsqlCommand(sql, connection, transaction);
command.Parameters.AddWithValue("artifact_id", artifactId);
command.Parameters.AddWithValue("component_id", componentId);
command.Parameters.AddWithValue("bom_ref", seed.BomRef);
command.Parameters.AddWithValue("scope", (object?)seed.Scope ?? DBNull.Value);
command.Parameters.AddWithValue("depth", seed.Depth);
command.Parameters.AddWithValue("introduced_via", (object?)seed.IntroducedVia ?? DBNull.Value);
var pathParameter = new NpgsqlParameter("dependency_path", NpgsqlDbType.Array | NpgsqlDbType.Text)
{
Value = (object?)seed.DependencyPath ?? DBNull.Value
};
command.Parameters.Add(pathParameter);
var rows = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
return rows > 0;
}
private static async Task IncrementComponentCountsAsync(
NpgsqlConnection connection,
NpgsqlTransaction transaction,
Guid componentId,
CancellationToken cancellationToken)
{
const string sql = """
UPDATE analytics.components
SET
artifact_count = artifact_count + 1,
sbom_count = sbom_count + 1,
last_seen_at = now(),
updated_at = now()
WHERE component_id = @component_id;
""";
await using var command = new NpgsqlCommand(sql, connection, transaction);
command.Parameters.AddWithValue("component_id", componentId);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private bool IsTenantAllowed(string tenant)
{
return TenantNormalizer.IsAllowed(tenant, _options.AllowedTenants);
}
private sealed record ContentPayload(byte[] Bytes, long Length, string Digest);
private sealed record ComponentSeed(
string BomRef,
string Purl,
string HashSha256,
string Name,
string? Version,
string ComponentType,
string? Supplier,
string? LicenseDeclared,
string? LicenseConcluded,
string? Description,
string? Cpe,
string? Scope,
string[]? DependencyPath,
int Depth,
string? IntroducedVia);
private sealed record ComponentKey(string Purl, string HashSha256);
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,126 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Platform.Analytics.Options;
namespace StellaOps.Platform.Analytics.Services;
public interface ICasContentReader
{
Task<CasContent?> OpenReadAsync(string casUri, CancellationToken cancellationToken);
}
public sealed record CasContent(Stream Stream, long? Length);
public sealed class FileCasContentReader : ICasContentReader
{
private readonly AnalyticsCasOptions _options;
private readonly ILogger<FileCasContentReader> _logger;
public FileCasContentReader(
IOptions<AnalyticsIngestionOptions> options,
ILogger<FileCasContentReader> logger)
{
_options = options?.Value.Cas ?? new AnalyticsCasOptions();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public Task<CasContent?> OpenReadAsync(string casUri, CancellationToken cancellationToken)
{
if (!TryParseCasUri(casUri, out var reference))
{
_logger.LogWarning("Unsupported CAS URI '{CasUri}'.", casUri);
return Task.FromResult<CasContent?>(null);
}
if (string.IsNullOrWhiteSpace(_options.RootPath))
{
_logger.LogWarning("CAS root path not configured; skipping {CasUri}.", casUri);
return Task.FromResult<CasContent?>(null);
}
var root = Path.GetFullPath(_options.RootPath);
foreach (var candidate in ExpandKeyCandidates(reference.Key))
{
var keyPath = candidate.Replace('/', Path.DirectorySeparatorChar);
var resolved = Path.GetFullPath(Path.Combine(root, reference.Bucket, keyPath));
if (!resolved.StartsWith(root, StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning("CAS URI '{CasUri}' resolved outside root '{Root}'.", casUri, root);
return Task.FromResult<CasContent?>(null);
}
if (!File.Exists(resolved))
{
continue;
}
var stream = new FileStream(resolved, FileMode.Open, FileAccess.Read, FileShare.Read);
var length = new FileInfo(resolved).Length;
return Task.FromResult<CasContent?>(new CasContent(stream, length));
}
_logger.LogWarning("CAS object not found at '{Key}' for '{CasUri}'.", reference.Key, casUri);
return Task.FromResult<CasContent?>(null);
}
private bool TryParseCasUri(string casUri, out CasReference reference)
{
reference = default!;
if (string.IsNullOrWhiteSpace(casUri))
{
return false;
}
if (!Uri.TryCreate(casUri, UriKind.Absolute, out var uri) ||
!string.Equals(uri.Scheme, "cas", StringComparison.OrdinalIgnoreCase))
{
return false;
}
var bucket = uri.Host;
var key = uri.AbsolutePath.TrimStart('/');
if (string.IsNullOrWhiteSpace(bucket))
{
if (string.IsNullOrWhiteSpace(_options.DefaultBucket))
{
return false;
}
bucket = _options.DefaultBucket!;
}
if (string.IsNullOrWhiteSpace(key))
{
return false;
}
reference = new CasReference(casUri, bucket, key);
return true;
}
private static IEnumerable<string> ExpandKeyCandidates(string key)
{
yield return key;
var colonIndex = key.IndexOf(':');
if (colonIndex <= 0 || colonIndex >= key.Length - 1)
{
yield break;
}
var prefix = key[..colonIndex];
var suffix = key[(colonIndex + 1)..];
yield return $"{prefix}/{suffix}";
yield return suffix;
}
}
public sealed record CasReference(string Uri, string Bucket, string Key);

View File

@@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Platform.Analytics.Services;
public interface IVulnerabilityCorrelationService
{
Task CorrelateForPurlsAsync(IReadOnlyCollection<string> purls, CancellationToken cancellationToken);
Task UpdateArtifactCountsAsync(Guid artifactId, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,603 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Npgsql;
using StellaOps.Messaging;
using StellaOps.Messaging.Abstractions;
using StellaOps.Platform.Analytics.Models;
using StellaOps.Platform.Analytics.Options;
using StellaOps.Platform.Analytics.Utilities;
namespace StellaOps.Platform.Analytics.Services;
public sealed class VulnerabilityCorrelationService : BackgroundService, IVulnerabilityCorrelationService
{
private readonly AnalyticsIngestionOptions _options;
private readonly AnalyticsIngestionDataSource _dataSource;
private readonly ILogger<VulnerabilityCorrelationService> _logger;
private readonly IEventStream<AdvisoryObservationUpdatedEvent>? _observationStream;
private readonly IEventStream<AdvisoryLinksetUpdatedEvent>? _linksetStream;
private readonly JsonSerializerOptions _jsonOptions = new()
{
PropertyNameCaseInsensitive = true
};
public VulnerabilityCorrelationService(
IOptions<AnalyticsIngestionOptions> options,
AnalyticsIngestionDataSource dataSource,
ILogger<VulnerabilityCorrelationService> logger,
IEventStreamFactory? eventStreamFactory = null)
{
_options = options?.Value ?? new AnalyticsIngestionOptions();
_options.Normalize();
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (eventStreamFactory is not null)
{
if (!string.IsNullOrWhiteSpace(_options.Streams.ConcelierObservationStream))
{
_observationStream = eventStreamFactory.Create<AdvisoryObservationUpdatedEvent>(
new EventStreamOptions
{
StreamName = _options.Streams.ConcelierObservationStream
});
}
if (!string.IsNullOrWhiteSpace(_options.Streams.ConcelierLinksetStream))
{
_linksetStream = eventStreamFactory.Create<AdvisoryLinksetUpdatedEvent>(
new EventStreamOptions
{
StreamName = _options.Streams.ConcelierLinksetStream
});
}
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (!_options.Enabled)
{
_logger.LogInformation("Vulnerability correlation disabled by configuration.");
return;
}
if (_observationStream is null && _linksetStream is null)
{
_logger.LogWarning("Vulnerability correlation disabled: no event streams configured.");
return;
}
var tasks = new List<Task>(2);
if (_observationStream is not null)
{
tasks.Add(ConsumeObservationStreamAsync(stoppingToken));
}
if (_linksetStream is not null)
{
tasks.Add(ConsumeLinksetStreamAsync(stoppingToken));
}
try
{
await Task.WhenAll(tasks).ConfigureAwait(false);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
_logger.LogInformation("Vulnerability correlation stopped.");
}
catch (Exception ex)
{
_logger.LogError(ex, "Vulnerability correlation failed.");
throw;
}
}
public async Task CorrelateForPurlsAsync(
IReadOnlyCollection<string> purls,
CancellationToken cancellationToken)
{
var normalized = NormalizePurls(purls);
if (normalized.Count == 0)
{
return;
}
await using var connection = await _dataSource
.OpenConnectionAsync(cancellationToken)
.ConfigureAwait(false);
if (connection is null)
{
_logger.LogWarning("Vulnerability correlation skipped: Postgres not configured.");
return;
}
var components = await LoadComponentsAsync(connection, normalized, cancellationToken)
.ConfigureAwait(false);
if (components.Count == 0)
{
return;
}
var matches = await LoadVulnerabilityMatchesAsync(connection, normalized, cancellationToken)
.ConfigureAwait(false);
if (matches.Count == 0)
{
return;
}
await using var transaction = await connection.BeginTransactionAsync(cancellationToken)
.ConfigureAwait(false);
foreach (var component in components)
{
if (!matches.TryGetValue(component.Purl, out var vulnMatches))
{
continue;
}
foreach (var match in vulnMatches)
{
if (!VulnerabilityCorrelationRules.TryParseNormalizedVersions(
match.NormalizedVersionsJson,
_jsonOptions,
out var rules,
out var error))
{
_logger.LogWarning(error, "Failed to parse normalized versions payload.");
}
var affects = rules.Count == 0
|| VersionRuleEvaluator.Matches(component.Version, rules);
var fixedVersion = VulnerabilityCorrelationRules.ExtractFixedVersion(rules);
var fixAvailable = !string.IsNullOrWhiteSpace(fixedVersion);
var affectedVersions = match.NormalizedVersionsJson;
await UpsertComponentVulnAsync(
connection,
transaction,
component.ComponentId,
match,
affects,
affectedVersions,
fixedVersion,
fixAvailable,
cancellationToken).ConfigureAwait(false);
}
}
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
}
public async Task UpdateArtifactCountsAsync(Guid artifactId, CancellationToken cancellationToken)
{
await using var connection = await _dataSource
.OpenConnectionAsync(cancellationToken)
.ConfigureAwait(false);
if (connection is null)
{
_logger.LogWarning("Artifact count update skipped: Postgres not configured.");
return;
}
const string sql = """
WITH counts AS (
SELECT
COUNT(DISTINCT cv.vuln_id) FILTER (WHERE cv.affects = TRUE) AS total,
COUNT(DISTINCT CASE WHEN cv.affects = TRUE AND cv.severity = 'critical' THEN cv.vuln_id END) AS critical,
COUNT(DISTINCT CASE WHEN cv.affects = TRUE AND cv.severity = 'high' THEN cv.vuln_id END) AS high,
COUNT(DISTINCT CASE WHEN cv.affects = TRUE AND cv.severity = 'medium' THEN cv.vuln_id END) AS medium,
COUNT(DISTINCT CASE WHEN cv.affects = TRUE AND cv.severity = 'low' THEN cv.vuln_id END) AS low
FROM analytics.artifact_components ac
JOIN analytics.component_vulns cv ON cv.component_id = ac.component_id
WHERE ac.artifact_id = @artifact_id
)
UPDATE analytics.artifacts
SET
vulnerability_count = COALESCE((SELECT total FROM counts), 0),
critical_count = COALESCE((SELECT critical FROM counts), 0),
high_count = COALESCE((SELECT high FROM counts), 0),
medium_count = COALESCE((SELECT medium FROM counts), 0),
low_count = COALESCE((SELECT low FROM counts), 0),
updated_at = now()
WHERE artifact_id = @artifact_id;
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("artifact_id", artifactId);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private async Task ConsumeObservationStreamAsync(CancellationToken stoppingToken)
{
if (_observationStream is null)
{
return;
}
var position = _options.Streams.StartFromBeginning
? StreamPosition.Beginning
: StreamPosition.End;
_logger.LogInformation(
"Subscribed to {StreamName} for advisory observation updates from {Position}.",
_observationStream.StreamName,
position.Value);
await foreach (var streamEvent in _observationStream.SubscribeAsync(position, stoppingToken))
{
var payload = streamEvent.Event;
if (!IsTenantAllowed(payload.TenantId))
{
continue;
}
var purls = payload.LinksetSummary.Purls?.ToArray() ?? Array.Empty<string>();
if (purls.Length == 0)
{
continue;
}
await CorrelateForPurlsAsync(purls, stoppingToken).ConfigureAwait(false);
await UpdateArtifactCountsForPurlsAsync(purls, stoppingToken).ConfigureAwait(false);
}
}
private async Task ConsumeLinksetStreamAsync(CancellationToken stoppingToken)
{
if (_linksetStream is null)
{
return;
}
var position = _options.Streams.StartFromBeginning
? StreamPosition.Beginning
: StreamPosition.End;
_logger.LogInformation(
"Subscribed to {StreamName} for advisory linkset updates from {Position}.",
_linksetStream.StreamName,
position.Value);
await foreach (var streamEvent in _linksetStream.SubscribeAsync(position, stoppingToken))
{
var payload = streamEvent.Event;
if (!IsTenantAllowed(payload.TenantId))
{
continue;
}
var purls = await ResolvePurlsForAdvisoryAsync(payload.AdvisoryId, stoppingToken)
.ConfigureAwait(false);
if (purls.Count == 0)
{
continue;
}
await CorrelateForPurlsAsync(purls, stoppingToken).ConfigureAwait(false);
await UpdateArtifactCountsForPurlsAsync(purls, stoppingToken).ConfigureAwait(false);
}
}
private bool IsTenantAllowed(string tenant)
=> TenantNormalizer.IsAllowed(tenant, _options.AllowedTenants);
private IReadOnlyList<string> NormalizePurls(IReadOnlyCollection<string> purls)
{
if (purls.Count == 0)
{
return Array.Empty<string>();
}
return purls
.Where(value => !string.IsNullOrWhiteSpace(value))
.Select(value => PurlParser.Parse(value).Normalized)
.Where(value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
}
private async Task<List<ComponentSnapshot>> LoadComponentsAsync(
NpgsqlConnection connection,
IReadOnlyList<string> purls,
CancellationToken cancellationToken)
{
const string sql = """
SELECT component_id, purl, COALESCE(NULLIF(purl_version, ''), NULLIF(version, ''))
FROM analytics.components
WHERE purl = ANY(@purls);
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("purls", purls);
var components = new List<ComponentSnapshot>();
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var componentId = reader.GetGuid(0);
var purl = reader.GetString(1);
var version = reader.IsDBNull(2) ? null : reader.GetString(2);
components.Add(new ComponentSnapshot(componentId, purl, version));
}
return components;
}
private async Task<Dictionary<string, List<VulnerabilityMatch>>> LoadVulnerabilityMatchesAsync(
NpgsqlConnection connection,
IReadOnlyList<string> purls,
CancellationToken cancellationToken)
{
const string sql = """
SELECT DISTINCT ON (aff.package_purl, adv.primary_vuln_id)
aff.package_purl,
adv.primary_vuln_id,
COALESCE(src.source_type, src.key, 'unknown') AS source,
adv.severity,
adv.published_at,
cvss.base_score,
cvss.vector,
canon.epss_score,
(kev.cve_id IS NOT NULL) AS kev_listed,
aff.normalized_versions::text AS normalized_versions
FROM vuln.advisory_affected aff
JOIN vuln.advisories adv ON adv.id = aff.advisory_id
LEFT JOIN vuln.sources src ON src.id = adv.source_id
LEFT JOIN LATERAL (
SELECT base_score, vector
FROM vuln.advisory_cvss
WHERE advisory_id = adv.id
ORDER BY is_primary DESC, base_score DESC, version DESC
LIMIT 1
) cvss ON TRUE
LEFT JOIN vuln.kev_flags kev ON kev.cve_id = adv.primary_vuln_id
LEFT JOIN vuln.advisory_canonical canon ON canon.cve = adv.primary_vuln_id
WHERE aff.package_purl = ANY(@purls)
AND adv.state = 'active'
ORDER BY aff.package_purl, adv.primary_vuln_id, COALESCE(src.priority, 100) ASC,
COALESCE(adv.updated_at, adv.created_at) DESC;
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("purls", purls);
var matches = new Dictionary<string, List<VulnerabilityMatch>>(StringComparer.OrdinalIgnoreCase);
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var purl = reader.GetString(0);
var vulnId = reader.GetString(1);
var source = reader.IsDBNull(2) ? "unknown" : reader.GetString(2);
var severity = reader.IsDBNull(3) ? null : reader.GetString(3);
var publishedAt = reader.IsDBNull(4) ? (DateTimeOffset?)null : reader.GetFieldValue<DateTimeOffset>(4);
var cvssScore = reader.IsDBNull(5) ? (decimal?)null : reader.GetDecimal(5);
var cvssVector = reader.IsDBNull(6) ? null : reader.GetString(6);
var epssScore = reader.IsDBNull(7) ? (decimal?)null : reader.GetDecimal(7);
var kevListed = !reader.IsDBNull(8) && reader.GetBoolean(8);
var normalizedVersionsJson = reader.IsDBNull(9) ? null : reader.GetString(9);
var match = new VulnerabilityMatch(
purl,
vulnId,
VulnerabilityCorrelationRules.NormalizeSource(source),
VulnerabilityCorrelationRules.NormalizeSeverity(severity),
cvssScore,
cvssVector,
epssScore,
kevListed,
normalizedVersionsJson,
publishedAt);
if (!matches.TryGetValue(purl, out var list))
{
list = new List<VulnerabilityMatch>();
matches[purl] = list;
}
list.Add(match);
}
return matches;
}
private async Task<IReadOnlyList<string>> ResolvePurlsForAdvisoryAsync(
string advisoryId,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(advisoryId))
{
return Array.Empty<string>();
}
await using var connection = await _dataSource
.OpenConnectionAsync(cancellationToken)
.ConfigureAwait(false);
if (connection is null)
{
return Array.Empty<string>();
}
const string sql = """
SELECT DISTINCT aff.package_purl
FROM vuln.advisory_affected aff
JOIN vuln.advisories adv ON adv.id = aff.advisory_id
WHERE aff.package_purl IS NOT NULL
AND (adv.primary_vuln_id = @advisory_id OR adv.advisory_key = @advisory_id);
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("advisory_id", advisoryId);
var purls = new List<string>();
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var purl = reader.GetString(0);
if (!string.IsNullOrWhiteSpace(purl))
{
purls.Add(purl);
}
}
return purls;
}
private async Task UpdateArtifactCountsForPurlsAsync(
IReadOnlyCollection<string> purls,
CancellationToken cancellationToken)
{
var normalized = NormalizePurls(purls);
if (normalized.Count == 0)
{
return;
}
await using var connection = await _dataSource
.OpenConnectionAsync(cancellationToken)
.ConfigureAwait(false);
if (connection is null)
{
return;
}
const string sql = """
WITH target_artifacts AS (
SELECT DISTINCT ac.artifact_id
FROM analytics.artifact_components ac
JOIN analytics.components c ON c.component_id = ac.component_id
WHERE c.purl = ANY(@purls)
),
counts AS (
SELECT
ac.artifact_id,
COUNT(DISTINCT cv.vuln_id) FILTER (WHERE cv.affects = TRUE) AS total,
COUNT(DISTINCT CASE WHEN cv.affects = TRUE AND cv.severity = 'critical' THEN cv.vuln_id END) AS critical,
COUNT(DISTINCT CASE WHEN cv.affects = TRUE AND cv.severity = 'high' THEN cv.vuln_id END) AS high,
COUNT(DISTINCT CASE WHEN cv.affects = TRUE AND cv.severity = 'medium' THEN cv.vuln_id END) AS medium,
COUNT(DISTINCT CASE WHEN cv.affects = TRUE AND cv.severity = 'low' THEN cv.vuln_id END) AS low
FROM analytics.artifact_components ac
JOIN analytics.component_vulns cv ON cv.component_id = ac.component_id
WHERE ac.artifact_id IN (SELECT artifact_id FROM target_artifacts)
GROUP BY ac.artifact_id
)
UPDATE analytics.artifacts a
SET
vulnerability_count = COALESCE(c.total, 0),
critical_count = COALESCE(c.critical, 0),
high_count = COALESCE(c.high, 0),
medium_count = COALESCE(c.medium, 0),
low_count = COALESCE(c.low, 0),
updated_at = now()
FROM target_artifacts t
LEFT JOIN counts c ON c.artifact_id = t.artifact_id
WHERE a.artifact_id = t.artifact_id;
""";
await using var command = new NpgsqlCommand(sql, connection);
command.Parameters.AddWithValue("purls", normalized);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private static async Task UpsertComponentVulnAsync(
NpgsqlConnection connection,
NpgsqlTransaction transaction,
Guid componentId,
VulnerabilityMatch match,
bool affects,
string? affectedVersions,
string? fixedVersion,
bool fixAvailable,
CancellationToken cancellationToken)
{
const string sql = """
INSERT INTO analytics.component_vulns (
component_id,
vuln_id,
source,
severity,
cvss_score,
cvss_vector,
epss_score,
kev_listed,
affects,
affected_versions,
fixed_version,
fix_available,
introduced_via,
published_at
)
VALUES (
@component_id,
@vuln_id,
@source,
@severity,
@cvss_score,
@cvss_vector,
@epss_score,
@kev_listed,
@affects,
@affected_versions,
@fixed_version,
@fix_available,
@introduced_via,
@published_at
)
ON CONFLICT (component_id, vuln_id) DO UPDATE SET
source = EXCLUDED.source,
severity = EXCLUDED.severity,
cvss_score = EXCLUDED.cvss_score,
cvss_vector = EXCLUDED.cvss_vector,
epss_score = EXCLUDED.epss_score,
kev_listed = EXCLUDED.kev_listed,
affects = EXCLUDED.affects,
affected_versions = EXCLUDED.affected_versions,
fixed_version = COALESCE(EXCLUDED.fixed_version, analytics.component_vulns.fixed_version),
fix_available = EXCLUDED.fix_available,
introduced_via = COALESCE(EXCLUDED.introduced_via, analytics.component_vulns.introduced_via),
published_at = COALESCE(EXCLUDED.published_at, analytics.component_vulns.published_at),
updated_at = now();
""";
await using var command = new NpgsqlCommand(sql, connection, transaction);
command.Parameters.AddWithValue("component_id", componentId);
command.Parameters.AddWithValue("vuln_id", match.VulnId);
command.Parameters.AddWithValue("source", match.Source);
command.Parameters.AddWithValue("severity", match.Severity);
command.Parameters.AddWithValue("cvss_score", (object?)match.CvssScore ?? DBNull.Value);
command.Parameters.AddWithValue("cvss_vector", (object?)match.CvssVector ?? DBNull.Value);
command.Parameters.AddWithValue("epss_score", (object?)match.EpssScore ?? DBNull.Value);
command.Parameters.AddWithValue("kev_listed", match.KevListed);
command.Parameters.AddWithValue("affects", affects);
command.Parameters.AddWithValue("affected_versions", (object?)affectedVersions ?? DBNull.Value);
command.Parameters.AddWithValue("fixed_version", (object?)fixedVersion ?? DBNull.Value);
command.Parameters.AddWithValue("fix_available", fixAvailable);
command.Parameters.AddWithValue("introduced_via", DBNull.Value);
command.Parameters.AddWithValue("published_at", (object?)match.PublishedAt ?? DBNull.Value);
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private sealed record ComponentSnapshot(Guid ComponentId, string Purl, string? Version);
private sealed record VulnerabilityMatch(
string Purl,
string VulnId,
string Source,
string Severity,
decimal? CvssScore,
string? CvssVector,
decimal? EpssScore,
bool KevListed,
string? NormalizedVersionsJson,
DateTimeOffset? PublishedAt);
}

View File

@@ -0,0 +1,29 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<RootNamespace>StellaOps.Platform.Analytics</RootNamespace>
<AssemblyName>StellaOps.Platform.Analytics</AssemblyName>
<Description>Analytics ingestion services for SBOM, vulnerability, and attestation data.</Description>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="Npgsql" />
<PackageReference Include="NuGet.Versioning" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="StellaOps.Platform.Analytics.Tests" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Concelier\__Libraries\StellaOps.Concelier.SbomIntegration\StellaOps.Concelier.SbomIntegration.csproj" />
<ProjectReference Include="..\..\Scanner\__Libraries\StellaOps.Scanner.Surface.FS\StellaOps.Scanner.Surface.FS.csproj" />
<ProjectReference Include="..\..\Router\__Libraries\StellaOps.Messaging\StellaOps.Messaging.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,88 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Concelier.SbomIntegration.Models;
namespace StellaOps.Platform.Analytics.Utilities;
public static class LicenseExpressionRenderer
{
public static string? BuildExpression(IReadOnlyList<ParsedLicense> licenses)
{
if (licenses is null || licenses.Count == 0)
{
return null;
}
var tokens = new List<string>();
foreach (var license in licenses)
{
if (license.Expression is not null)
{
var expression = Render(license.Expression);
if (!string.IsNullOrWhiteSpace(expression))
{
tokens.Add(expression);
}
continue;
}
if (!string.IsNullOrWhiteSpace(license.SpdxId))
{
tokens.Add(license.SpdxId.Trim());
continue;
}
if (!string.IsNullOrWhiteSpace(license.Name))
{
tokens.Add(license.Name.Trim());
}
}
if (tokens.Count == 0)
{
return null;
}
return string.Join(" OR ", tokens);
}
public static string Render(ParsedLicenseExpression expression)
{
return expression switch
{
SimpleLicense simple => simple.Id,
OrLater later => $"{later.LicenseId}+",
WithException withException => $"{RenderNode(withException.License, true)} WITH {withException.Exception}",
ConjunctiveSet conjunctive => RenderGroup(conjunctive.Members, " AND "),
DisjunctiveSet disjunctive => RenderGroup(disjunctive.Members, " OR "),
_ => string.Empty
};
}
private static string RenderGroup(ImmutableArray<ParsedLicenseExpression> members, string separator)
{
var rendered = members
.Select(member => RenderNode(member, false))
.Where(value => !string.IsNullOrWhiteSpace(value))
.ToArray();
return string.Join(separator, rendered);
}
private static string RenderNode(ParsedLicenseExpression expression, bool wrapSets)
{
return expression switch
{
ConjunctiveSet conjunctive => wrapSets
? $"({RenderGroup(conjunctive.Members, " AND ")})"
: RenderGroup(conjunctive.Members, " AND "),
DisjunctiveSet disjunctive => wrapSets
? $"({RenderGroup(disjunctive.Members, " OR ")})"
: RenderGroup(disjunctive.Members, " OR "),
WithException withException => $"{RenderNode(withException.License, true)} WITH {withException.Exception}",
OrLater later => $"{later.LicenseId}+",
SimpleLicense simple => simple.Id,
_ => string.Empty
};
}
}

View File

@@ -0,0 +1,199 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace StellaOps.Platform.Analytics.Utilities;
public sealed record PurlIdentity(
string Raw,
string Normalized,
string? Type,
string? Namespace,
string? Name,
string? Version);
public static class PurlParser
{
private static readonly HashSet<string> StrippedQualifiers = new(StringComparer.OrdinalIgnoreCase)
{
"arch",
"architecture",
"os",
"platform",
"type",
"classifier",
"checksum",
"download_url",
"vcs_url",
"repository_url"
};
private static readonly Regex Pattern = new(
@"^pkg:([a-zA-Z][a-zA-Z0-9+.-]*)(?:/([^/@#?]+))?/([^/@#?]+)(?:@([^?#]+))?(?:\?([^#]+))?(?:#(.+))?$",
RegexOptions.Compiled);
public static PurlIdentity Parse(string? purl)
{
if (string.IsNullOrWhiteSpace(purl))
{
return new PurlIdentity(string.Empty, string.Empty, null, null, null, null);
}
var trimmed = purl.Trim();
if (!trimmed.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase))
{
var lowered = trimmed.ToLowerInvariant();
return new PurlIdentity(trimmed, lowered, null, null, lowered, null);
}
var match = Pattern.Match(trimmed);
if (!match.Success)
{
var lowered = trimmed.ToLowerInvariant();
return new PurlIdentity(trimmed, lowered, null, null, lowered, null);
}
var type = match.Groups[1].Value.ToLowerInvariant();
var ns = match.Groups[2].Success ? NormalizeNamespace(match.Groups[2].Value, type) : null;
var name = NormalizeName(match.Groups[3].Value, type);
var version = match.Groups[4].Success ? Decode(match.Groups[4].Value) : null;
var qualifiers = match.Groups[5].Success ? NormalizeQualifiers(match.Groups[5].Value) : null;
var normalized = BuildPurl(type, ns, name, version, qualifiers);
return new PurlIdentity(trimmed, normalized, type, ns, name, version);
}
public static string BuildGeneric(string name, string? version)
{
var safeName = string.IsNullOrWhiteSpace(name) ? "unknown" : Uri.EscapeDataString(name.Trim());
if (string.IsNullOrWhiteSpace(version))
{
return $"pkg:generic/{safeName}";
}
var safeVersion = Uri.EscapeDataString(version.Trim());
return $"pkg:generic/{safeName}@{safeVersion}";
}
private static string NormalizeNamespace(string ns, string type)
{
var decoded = Decode(ns);
if (type == "npm" && decoded.StartsWith("@", StringComparison.Ordinal))
{
decoded = decoded.ToLowerInvariant();
return Uri.EscapeDataString(decoded);
}
return decoded.ToLowerInvariant();
}
private static string NormalizeName(string name, string type)
{
var decoded = Decode(name);
return type switch
{
"golang" => decoded,
"nuget" => decoded.ToLowerInvariant(),
_ => decoded.ToLowerInvariant()
};
}
private static string? NormalizeQualifiers(string qualifiers)
{
if (string.IsNullOrWhiteSpace(qualifiers))
{
return null;
}
var pairs = qualifiers
.Split('&', StringSplitOptions.RemoveEmptyEntries)
.Select(static pair =>
{
var eqIndex = pair.IndexOf('=');
if (eqIndex < 0)
{
return (Key: Decode(pair).ToLowerInvariant(), Value: (string?)null);
}
var key = Decode(pair[..eqIndex]).ToLowerInvariant();
var value = Decode(pair[(eqIndex + 1)..]);
return (Key: key, Value: value);
})
.Where(pair => !StrippedQualifiers.Contains(pair.Key))
.OrderBy(pair => pair.Key, StringComparer.Ordinal)
.ToList();
if (pairs.Count == 0)
{
return null;
}
var builder = new StringBuilder();
for (var i = 0; i < pairs.Count; i++)
{
var (key, value) = pairs[i];
if (i > 0)
{
builder.Append('&');
}
builder.Append(key);
if (!string.IsNullOrEmpty(value))
{
builder.Append('=');
builder.Append(value);
}
}
return builder.ToString();
}
private static string BuildPurl(
string type,
string? ns,
string name,
string? version,
string? qualifiers)
{
var builder = new StringBuilder("pkg:");
builder.Append(type);
builder.Append('/');
if (!string.IsNullOrWhiteSpace(ns))
{
builder.Append(ns);
builder.Append('/');
}
builder.Append(name);
if (!string.IsNullOrWhiteSpace(version))
{
builder.Append('@');
builder.Append(version);
}
if (!string.IsNullOrWhiteSpace(qualifiers))
{
builder.Append('?');
builder.Append(qualifiers);
}
return builder.ToString();
}
private static string Decode(string value)
{
try
{
return Uri.UnescapeDataString(value);
}
catch (InvalidOperationException)
{
return value;
}
}
}

View File

@@ -0,0 +1,20 @@
using System.Security.Cryptography;
using System.Text;
namespace StellaOps.Platform.Analytics.Utilities;
public static class Sha256Hasher
{
public static string Compute(string value)
{
var bytes = Encoding.UTF8.GetBytes(value);
var hash = SHA256.HashData(bytes);
return $"sha256:{Convert.ToHexStringLower(hash)}";
}
public static string Compute(byte[] value)
{
var hash = SHA256.HashData(value);
return $"sha256:{Convert.ToHexStringLower(hash)}";
}
}

View File

@@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Platform.Analytics.Utilities;
public static class TenantNormalizer
{
public static string Normalize(string? tenant)
{
if (string.IsNullOrWhiteSpace(tenant))
{
return string.Empty;
}
var trimmed = tenant.Trim();
if (trimmed.StartsWith("urn:tenant:", StringComparison.OrdinalIgnoreCase))
{
trimmed = trimmed["urn:tenant:".Length..];
}
return trimmed;
}
public static bool IsAllowed(string tenant, IReadOnlyCollection<string> allowedTenants)
{
if (allowedTenants is null || allowedTenants.Count == 0)
{
return true;
}
var normalized = Normalize(tenant);
foreach (var allowed in allowedTenants)
{
if (string.Equals(Normalize(allowed), normalized, StringComparison.OrdinalIgnoreCase))
{
return true;
}
}
return false;
}
}

View File

@@ -0,0 +1,162 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
using NuGet.Versioning;
namespace StellaOps.Platform.Analytics.Utilities;
public sealed record NormalizedVersionRule
{
[JsonPropertyName("scheme")]
public string? Scheme { get; init; }
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("min")]
public string? Min { get; init; }
[JsonPropertyName("minInclusive")]
public bool? MinInclusive { get; init; }
[JsonPropertyName("max")]
public string? Max { get; init; }
[JsonPropertyName("maxInclusive")]
public bool? MaxInclusive { get; init; }
[JsonPropertyName("value")]
public string? Value { get; init; }
[JsonPropertyName("notes")]
public string? Notes { get; init; }
}
public static class VersionRuleEvaluator
{
public static bool Matches(string? version, IReadOnlyList<NormalizedVersionRule> rules)
{
if (string.IsNullOrWhiteSpace(version))
{
return false;
}
if (rules is null || rules.Count == 0)
{
return true;
}
foreach (var rule in rules)
{
if (Matches(version, rule))
{
return true;
}
}
return false;
}
public static bool Matches(string? version, NormalizedVersionRule? rule)
{
if (rule is null || string.IsNullOrWhiteSpace(version))
{
return false;
}
var scheme = rule.Scheme?.Trim().ToLowerInvariant();
if (!string.Equals(scheme, "semver", StringComparison.Ordinal))
{
return MatchesExact(version, rule);
}
if (!NuGetVersion.TryParse(version, out var componentVersion))
{
return false;
}
var type = rule.Type?.Trim().ToLowerInvariant();
switch (type)
{
case "exact":
return TryParseVersion(rule.Value ?? rule.Min ?? rule.Max, out var exact)
&& exact is not null
&& componentVersion == exact;
case "range":
return TryBuildRange(rule, out var range) && range.Satisfies(componentVersion);
case "lt":
return TryBuildRange(rule with { MaxInclusive = false }, out range) && range.Satisfies(componentVersion);
case "lte":
return TryBuildRange(rule with { MaxInclusive = true }, out range) && range.Satisfies(componentVersion);
case "gt":
return TryBuildRange(rule with { MinInclusive = false }, out range) && range.Satisfies(componentVersion);
case "gte":
return TryBuildRange(rule with { MinInclusive = true }, out range) && range.Satisfies(componentVersion);
default:
return false;
}
}
private static bool MatchesExact(string version, NormalizedVersionRule rule)
{
if (!string.Equals(rule.Type?.Trim(), "exact", StringComparison.OrdinalIgnoreCase))
{
return false;
}
var target = rule.Value ?? rule.Min ?? rule.Max;
return !string.IsNullOrWhiteSpace(target)
&& string.Equals(target.Trim(), version.Trim(), StringComparison.OrdinalIgnoreCase);
}
private static bool TryBuildRange(NormalizedVersionRule rule, out VersionRange range)
{
range = VersionRange.All;
var hasMin = TryParseVersion(rule.Min, out var min);
var hasMax = TryParseVersion(rule.Max, out var max);
if (!hasMin && !hasMax)
{
return false;
}
if (!hasMin)
{
range = new VersionRange(
minVersion: null,
includeMinVersion: false,
maxVersion: max,
includeMaxVersion: rule.MaxInclusive ?? false);
return true;
}
if (!hasMax)
{
range = new VersionRange(
minVersion: min,
includeMinVersion: rule.MinInclusive ?? true,
maxVersion: null,
includeMaxVersion: false);
return true;
}
range = new VersionRange(
minVersion: min,
includeMinVersion: rule.MinInclusive ?? true,
maxVersion: max,
includeMaxVersion: rule.MaxInclusive ?? false);
return true;
}
private static bool TryParseVersion(string? value, out NuGetVersion? version)
{
if (!string.IsNullOrWhiteSpace(value) && NuGetVersion.TryParse(value, out version))
{
return true;
}
version = null;
return false;
}
}

View File

@@ -0,0 +1,89 @@
// SPDX-License-Identifier: BUSL-1.1
// Copyright (c) 2026 stella-ops.org
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Platform.Analytics.Utilities;
internal static class VulnerabilityCorrelationRules
{
public static bool TryParseNormalizedVersions(
string? json,
JsonSerializerOptions options,
out IReadOnlyList<NormalizedVersionRule> rules,
out Exception? error)
{
error = null;
if (string.IsNullOrWhiteSpace(json) || json == "[]")
{
rules = Array.Empty<NormalizedVersionRule>();
return true;
}
try
{
var parsed = JsonSerializer.Deserialize<List<NormalizedVersionRule>>(json, options);
rules = parsed?.Where(rule => rule is not null).ToArray()
?? Array.Empty<NormalizedVersionRule>();
return true;
}
catch (JsonException ex)
{
rules = Array.Empty<NormalizedVersionRule>();
error = ex;
return false;
}
}
public static string NormalizeSeverity(string? severity)
{
if (string.IsNullOrWhiteSpace(severity))
{
return "unknown";
}
return severity.Trim().ToLowerInvariant() switch
{
"critical" => "critical",
"high" => "high",
"medium" => "medium",
"low" => "low",
"none" => "none",
_ => "unknown"
};
}
public static string NormalizeSource(string? source)
{
if (string.IsNullOrWhiteSpace(source))
{
return "unknown";
}
return source.Trim().ToLowerInvariant();
}
public static string? ExtractFixedVersion(IReadOnlyList<NormalizedVersionRule> rules)
{
if (rules.Count == 0)
{
return null;
}
foreach (var rule in rules)
{
var type = rule.Type?.Trim().ToLowerInvariant();
if (!string.IsNullOrWhiteSpace(rule.Max) &&
(type == "lt" || type == "lte" || type == "range"))
{
return rule.Max;
}
}
return null;
}
}