feat: Implement MongoDB orchestrator storage with registry, commands, and heartbeats
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added NullAdvisoryObservationEventTransport for handling advisory observation events.
- Created IOrchestratorRegistryStore interface for orchestrator registry operations.
- Implemented MongoOrchestratorRegistryStore for MongoDB interactions with orchestrator data.
- Defined OrchestratorCommandDocument and OrchestratorCommandRecord for command handling.
- Added OrchestratorHeartbeatDocument and OrchestratorHeartbeatRecord for heartbeat tracking.
- Created OrchestratorRegistryDocument and OrchestratorRegistryRecord for registry management.
- Developed tests for orchestrator collections migration and MongoOrchestratorRegistryStore functionality.
- Introduced AirgapImportRequest and AirgapImportValidator for air-gapped VEX bundle imports.
- Added incident mode rules sample JSON for notifier configuration.
This commit is contained in:
StellaOps Bot
2025-11-22 12:35:38 +02:00
parent cbdc05b24d
commit f43e828b4e
96 changed files with 3425 additions and 976 deletions

View File

@@ -1,4 +1,4 @@
# Concelier · AGENTS Charter (Sprint 01120113)
# Concelier · AGENTS Charter (Sprint 01120114)
## Module Scope & Working Directory
- Working directory: `src/Concelier/**` (WebService, __Libraries, Storage.Mongo, analyzers, tests, seed-data). Do not edit other modules unless explicitly referenced by this sprint.
@@ -17,7 +17,9 @@
- `docs/modules/concelier/architecture.md`
- `docs/modules/concelier/link-not-merge-schema.md`
- `docs/provenance/inline-dsse.md` (for provenance anchors/DSSE notes)
- Any sprint-specific ADRs/notes linked from `docs/implplan/SPRINT_0112_0001_0001_concelier_i.md` or `SPRINT_0113_0001_0002_concelier_ii.md`.
- `docs/modules/concelier/prep/2025-11-22-oas-obs-prep.md` (OAS + observability prep)
- `docs/modules/concelier/prep/2025-11-20-orchestrator-registry-prep.md` (orchestrator registry/control contracts)
- Any sprint-specific ADRs/notes linked from `docs/implplan/SPRINT_0112_0001_0001_concelier_i.md`, `SPRINT_0113_0001_0002_concelier_ii.md`, or `SPRINT_0114_0001_0003_concelier_iii.md`.
## Working Agreements
- **Aggregation-Only Contract (AOC):** no derived semantics in ingestion; enforce via `AOCWriteGuard` and analyzers. Raw observations are append-only; linksets carry correlations/conflicts only.

View File

@@ -1,8 +1,9 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Concelier.RawModels;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Concelier.Core.Attestation;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.WebService.Contracts;
@@ -89,7 +90,8 @@ public sealed record AdvisoryRawListResponse(
public sealed record AdvisoryEvidenceResponse(
[property: JsonPropertyName("advisoryKey")] string AdvisoryKey,
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records);
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
[property: JsonPropertyName("attestation")] AttestationClaims? Attestation);
public sealed record AdvisoryRawProvenanceResponse(
[property: JsonPropertyName("id")] string Id,

View File

@@ -0,0 +1,56 @@
using System.ComponentModel.DataAnnotations;
using StellaOps.Concelier.Storage.Mongo.Orchestrator;
namespace StellaOps.Concelier.WebService.Contracts;
public sealed record OrchestratorRegistryRequest(
[property: Required] string ConnectorId,
[property: Required] string Source,
[property: Required] string[] Capabilities,
[property: Required] string AuthRef,
[property: Required] OrchestratorScheduleDto Schedule,
[property: Required] OrchestratorRatePolicyDto RatePolicy,
[property: Required] string[] ArtifactKinds,
[property: Required] string LockKey,
[property: Required] OrchestratorEgressGuardDto EgressGuard);
public sealed record OrchestratorScheduleDto(
string Cron,
string TimeZone,
int MaxParallelRuns,
int MaxLagMinutes);
public sealed record OrchestratorRatePolicyDto(
int Rpm,
int Burst,
int CooldownSeconds);
public sealed record OrchestratorEgressGuardDto(
string[] Allowlist,
bool AirgapMode);
public sealed record OrchestratorHeartbeatRequest(
[property: Required] string ConnectorId,
[property: Required] Guid RunId,
[property: Required] long Sequence,
[property: Required] OrchestratorHeartbeatStatus Status,
int? Progress,
int? QueueDepth,
string? LastArtifactHash,
string? LastArtifactKind,
string? ErrorCode,
int? RetryAfterSeconds,
DateTimeOffset? TimestampUtc);
public sealed record OrchestratorCommandRequest(
[property: Required] string ConnectorId,
[property: Required] Guid RunId,
[property: Required] long Sequence,
[property: Required] OrchestratorCommandKind Command,
OrchestratorThrottleOverrideDto? Throttle,
OrchestratorBackfillRangeDto? Backfill,
DateTimeOffset? ExpiresAt);
public sealed record OrchestratorThrottleOverrideDto(int? Rpm, int? Burst, int? CooldownSeconds, DateTimeOffset? ExpiresAt);
public sealed record OrchestratorBackfillRangeDto(string? FromCursor, string? ToCursor);

View File

@@ -21,6 +21,8 @@ public sealed class ConcelierOptions
public AdvisoryChunkOptions AdvisoryChunks { get; set; } = new();
public EvidenceBundleOptions Evidence { get; set; } = new();
public StellaOpsCryptoOptions Crypto { get; } = new();
public sealed class StorageOptions
@@ -172,4 +174,20 @@ public sealed class ConcelierOptions
public int CacheDurationSeconds { get; set; } = 30;
}
public sealed class EvidenceBundleOptions
{
public bool Enabled { get; set; } = true;
public string Root { get; set; } = System.IO.Path.Combine("out", "evidence", "bundles");
public string? DefaultManifestFileName { get; set; } = "manifest.json";
public string? DefaultTransparencyFileName { get; set; } = "transparency.json";
public string PipelineVersion { get; set; } = "git:unknown";
[JsonIgnore]
public string RootAbsolute { get; internal set; } = string.Empty;
}
}

View File

@@ -19,6 +19,7 @@ public static class ConcelierOptionsPostConfigure
options.Authority ??= new ConcelierOptions.AuthorityOptions();
options.Features ??= new ConcelierOptions.FeaturesOptions();
options.Evidence ??= new ConcelierOptions.EvidenceBundleOptions();
var authority = options.Authority;
if (string.IsNullOrWhiteSpace(authority.ClientSecret)
@@ -44,8 +45,8 @@ public static class ConcelierOptionsPostConfigure
authority.ClientSecret = secret;
}
options.Mirror ??= new ConcelierOptions.MirrorOptions();
var mirror = options.Mirror;
options.Mirror ??= new ConcelierOptions.MirrorOptions();
var mirror = options.Mirror;
if (string.IsNullOrWhiteSpace(mirror.ExportRoot))
{
@@ -65,9 +66,33 @@ public static class ConcelierOptionsPostConfigure
mirror.LatestDirectoryName = "latest";
}
if (string.IsNullOrWhiteSpace(mirror.MirrorDirectoryName))
{
mirror.MirrorDirectoryName = "mirror";
}
}
}
if (string.IsNullOrWhiteSpace(mirror.MirrorDirectoryName))
{
mirror.MirrorDirectoryName = "mirror";
}
var evidence = options.Evidence;
if (string.IsNullOrWhiteSpace(evidence.Root))
{
evidence.Root = Path.Combine("out", "evidence", "bundles");
}
var evidenceRoot = evidence.Root;
if (!Path.IsPathRooted(evidenceRoot))
{
evidenceRoot = Path.Combine(contentRootPath, evidenceRoot);
}
evidence.RootAbsolute = Path.GetFullPath(evidenceRoot);
if (string.IsNullOrWhiteSpace(evidence.DefaultManifestFileName))
{
evidence.DefaultManifestFileName = "manifest.json";
}
if (string.IsNullOrWhiteSpace(evidence.DefaultTransparencyFileName))
{
evidence.DefaultTransparencyFileName = "transparency.json";
}
}
}

View File

@@ -137,6 +137,9 @@ public static class ConcelierOptionsValidator
options.Mirror ??= new ConcelierOptions.MirrorOptions();
ValidateMirror(options.Mirror);
options.Evidence ??= new ConcelierOptions.EvidenceBundleOptions();
ValidateEvidence(options.Evidence);
options.AdvisoryChunks ??= new ConcelierOptions.AdvisoryChunkOptions();
ValidateAdvisoryChunks(options.AdvisoryChunks);
}
@@ -312,4 +315,22 @@ public static class ConcelierOptionsValidator
throw new InvalidOperationException("Advisory chunk cacheDurationSeconds must be greater than or equal to zero.");
}
}
private static void ValidateEvidence(ConcelierOptions.EvidenceBundleOptions evidence)
{
if (string.IsNullOrWhiteSpace(evidence.Root))
{
throw new InvalidOperationException("Evidence bundle root must be configured.");
}
if (string.IsNullOrWhiteSpace(evidence.RootAbsolute))
{
throw new InvalidOperationException("Evidence bundle root could not be resolved.");
}
if (string.IsNullOrWhiteSpace(evidence.PipelineVersion))
{
throw new InvalidOperationException("Evidence bundle pipelineVersion must be provided.");
}
}
}

View File

@@ -55,6 +55,7 @@ using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Aliases;
using StellaOps.Provenance.Mongo;
using StellaOps.Concelier.Core.Attestation;
using StellaOps.Concelier.Storage.Mongo.Orchestrator;
var builder = WebApplication.CreateBuilder(args);
@@ -398,6 +399,162 @@ app.MapGet("/.well-known/openapi", ([FromServices] OpenApiDiscoveryDocumentProvi
}
}).WithName("GetConcelierOpenApiDocument");
var orchestratorGroup = app.MapGroup("/internal/orch");
if (authorityConfigured)
{
orchestratorGroup.RequireAuthorization();
}
orchestratorGroup.MapPost("/registry", async (
HttpContext context,
[FromBody] OrchestratorRegistryRequest request,
[FromServices] IOrchestratorRegistryStore store,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(request.ConnectorId) || string.IsNullOrWhiteSpace(request.Source))
{
return Problem(context, "connectorId and source are required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId and source.");
}
var now = timeProvider.GetUtcNow();
var record = new OrchestratorRegistryRecord(
tenant,
request.ConnectorId.Trim(),
request.Source.Trim(),
request.Capabilities,
request.AuthRef,
new OrchestratorSchedule(
request.Schedule.Cron,
string.IsNullOrWhiteSpace(request.Schedule.TimeZone) ? "UTC" : request.Schedule.TimeZone,
request.Schedule.MaxParallelRuns,
request.Schedule.MaxLagMinutes),
new OrchestratorRatePolicy(request.RatePolicy.Rpm, request.RatePolicy.Burst, request.RatePolicy.CooldownSeconds),
request.ArtifactKinds,
request.LockKey,
new OrchestratorEgressGuard(request.EgressGuard.Allowlist, request.EgressGuard.AirgapMode),
now,
now);
await store.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
return Results.Accepted();
}).WithName("UpsertOrchestratorRegistry");
orchestratorGroup.MapPost("/heartbeat", async (
HttpContext context,
[FromBody] OrchestratorHeartbeatRequest request,
[FromServices] IOrchestratorRegistryStore store,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(request.ConnectorId))
{
return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId.");
}
if (request.Sequence < 0)
{
return Problem(context, "sequence must be non-negative", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide a non-negative sequence.");
}
var timestamp = request.TimestampUtc ?? timeProvider.GetUtcNow();
var heartbeat = new OrchestratorHeartbeatRecord(
tenant,
request.ConnectorId.Trim(),
request.RunId,
request.Sequence,
request.Status,
request.Progress,
request.QueueDepth,
request.LastArtifactHash,
request.LastArtifactKind,
request.ErrorCode,
request.RetryAfterSeconds,
timestamp);
await store.AppendHeartbeatAsync(heartbeat, cancellationToken).ConfigureAwait(false);
return Results.Accepted();
}).WithName("RecordOrchestratorHeartbeat");
orchestratorGroup.MapPost("/commands", async (
HttpContext context,
[FromBody] OrchestratorCommandRequest request,
[FromServices] IOrchestratorRegistryStore store,
CancellationToken cancellationToken) =>
{
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(request.ConnectorId))
{
return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId.");
}
if (request.Sequence < 0)
{
return Problem(context, "sequence must be non-negative", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide a non-negative sequence.");
}
var command = new OrchestratorCommandRecord(
tenant,
request.ConnectorId.Trim(),
request.RunId,
request.Sequence,
request.Command,
request.Throttle is null
? null
: new OrchestratorThrottleOverride(
request.Throttle.Rpm,
request.Throttle.Burst,
request.Throttle.CooldownSeconds,
request.Throttle.ExpiresAt),
request.Backfill is null
? null
: new OrchestratorBackfillRange(request.Backfill.FromCursor, request.Backfill.ToCursor),
DateTimeOffset.UtcNow,
request.ExpiresAt);
await store.EnqueueCommandAsync(command, cancellationToken).ConfigureAwait(false);
return Results.Accepted();
}).WithName("EnqueueOrchestratorCommand");
orchestratorGroup.MapGet("/commands", async (
HttpContext context,
[FromQuery] string connectorId,
[FromQuery] Guid runId,
[FromQuery] long? afterSequence,
[FromServices] IOrchestratorRegistryStore store,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(connectorId))
{
return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId.");
}
var commands = await store.GetPendingCommandsAsync(tenant, connectorId.Trim(), runId, afterSequence, cancellationToken).ConfigureAwait(false);
return Results.Ok(commands);
}).WithName("GetOrchestratorCommands");
var jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web);
jsonOptions.Converters.Add(new JsonStringEnumConverter());
@@ -836,6 +993,7 @@ var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKe
HttpContext context,
[FromServices] IAdvisoryRawService rawService,
[FromServices] EvidenceBundleAttestationBuilder attestationBuilder,
[FromServices] ILogger<Program> logger,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
@@ -879,10 +1037,16 @@ var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKe
record.Document))
.ToArray();
var evidenceOptions = resolvedConcelierOptions.Evidence ?? new ConcelierOptions.EvidenceBundleOptions();
var attestation = await TryBuildAttestationAsync(
context,
evidenceOptions,
attestationBuilder,
logger,
cancellationToken).ConfigureAwait(false);
var responseKey = recordResponses[0].Document.AdvisoryKey ?? canonicalKey;
var response = new AdvisoryEvidenceResponse(responseKey, recordResponses);
// TODO: Attach attestation metadata when Evidence Bundle tarball is available per tenant/advisory.
// The builder is registered for future use once bundle paths are discoverable from evidence storage.
var response = new AdvisoryEvidenceResponse(responseKey, recordResponses, attestation);
return JsonResult(response);
});
if (authorityConfigured)
@@ -1622,6 +1786,120 @@ static KeyValuePair<string, object?>[] BuildJobMetricTags(string jobKind, string
new KeyValuePair<string, object?>("job.outcome", outcome),
};
static async Task<AttestationClaims?> TryBuildAttestationAsync(
HttpContext context,
ConcelierOptions.EvidenceBundleOptions evidenceOptions,
EvidenceBundleAttestationBuilder builder,
ILogger logger,
CancellationToken cancellationToken)
{
var bundlePath = context.Request.Query.TryGetValue("bundlePath", out var bundleValues)
? bundleValues.FirstOrDefault()
: null;
if (string.IsNullOrWhiteSpace(bundlePath))
{
return null;
}
var manifestPath = context.Request.Query.TryGetValue("manifestPath", out var manifestValues)
? manifestValues.FirstOrDefault()
: null;
var transparencyPath = context.Request.Query.TryGetValue("transparencyPath", out var transparencyValues)
? transparencyValues.FirstOrDefault()
: null;
var pipelineVersion = context.Request.Query.TryGetValue("pipelineVersion", out var pipelineValues)
? pipelineValues.FirstOrDefault()
: null;
pipelineVersion = string.IsNullOrWhiteSpace(pipelineVersion)
? evidenceOptions.PipelineVersion
: pipelineVersion.Trim();
var root = evidenceOptions.RootAbsolute;
var resolvedBundlePath = ResolveEvidencePath(bundlePath, root);
if (string.IsNullOrWhiteSpace(resolvedBundlePath) || !File.Exists(resolvedBundlePath))
{
return null;
}
var resolvedManifestPath = string.IsNullOrWhiteSpace(manifestPath)
? ResolveSibling(resolvedBundlePath, evidenceOptions.DefaultManifestFileName)
: ResolveEvidencePath(manifestPath!, root);
if (string.IsNullOrWhiteSpace(resolvedManifestPath) || !File.Exists(resolvedManifestPath))
{
return null;
}
var resolvedTransparencyPath = string.IsNullOrWhiteSpace(transparencyPath)
? ResolveSibling(resolvedBundlePath, evidenceOptions.DefaultTransparencyFileName)
: ResolveEvidencePath(transparencyPath!, root);
try
{
return await builder.BuildAsync(
new EvidenceBundleAttestationRequest(
resolvedBundlePath!,
resolvedManifestPath!,
resolvedTransparencyPath,
pipelineVersion ?? "git:unknown"),
cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to build attestation for evidence bundle {BundlePath}", resolvedBundlePath);
return null;
}
}
static string? ResolveEvidencePath(string candidate, string root)
{
if (string.IsNullOrWhiteSpace(candidate))
{
return null;
}
var path = candidate;
if (!Path.IsPathRooted(path))
{
path = Path.Combine(root, path);
}
var fullPath = Path.GetFullPath(path);
if (!string.IsNullOrWhiteSpace(root))
{
var rootPath = Path.GetFullPath(root)
.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
if (!fullPath.StartsWith(rootPath, StringComparison.OrdinalIgnoreCase))
{
return null;
}
}
return fullPath;
}
static string? ResolveSibling(string? bundlePath, string? fileName)
{
if (string.IsNullOrWhiteSpace(bundlePath) || string.IsNullOrWhiteSpace(fileName))
{
return null;
}
var directory = Path.GetDirectoryName(bundlePath);
if (string.IsNullOrWhiteSpace(directory))
{
return null;
}
return Path.Combine(directory, fileName);
}
void ApplyNoCache(HttpResponse response)
{
if (response is null)

View File

@@ -39,7 +39,8 @@ public sealed record AdvisoryLinksetProvenance(
public sealed record AdvisoryLinksetConflict(
string Field,
string Reason,
IReadOnlyList<string>? Values);
IReadOnlyList<string>? Values,
IReadOnlyList<string>? SourceIds = null);
internal static class BsonDocumentHelper
{

View File

@@ -32,10 +32,23 @@ internal static class AdvisoryLinksetNormalization
ArgumentNullException.ThrowIfNull(linkset);
var normalized = Build(linkset.PackageUrls);
var conflicts = ExtractConflicts(linkset);
var confidence = ComputeConfidence(linkset, providedConfidence, conflicts);
return (normalized, confidence, conflicts);
var inputs = new[]
{
new LinksetCorrelation.Input(
Vendor: null,
FetchedAt: null,
Aliases: linkset.Aliases,
Purls: linkset.PackageUrls,
Cpes: linkset.Cpes,
References: linkset.References.Select(r => r.Url).ToArray())
};
var noteConflicts = ExtractConflicts(linkset);
var (confidenceScore, conflicts) = LinksetCorrelation.Compute(inputs, noteConflicts);
var coerced = providedConfidence.HasValue ? CoerceConfidence(providedConfidence) : confidenceScore;
return (normalized, coerced, conflicts);
}
private static AdvisoryLinksetNormalized? Build(IEnumerable<string> purlValues)
@@ -190,37 +203,4 @@ internal static class AdvisoryLinksetNormalization
return conflicts;
}
private static double? ComputeConfidence(RawLinkset linkset, double? providedConfidence, IReadOnlyList<AdvisoryLinksetConflict> conflicts)
{
if (providedConfidence.HasValue)
{
return CoerceConfidence(providedConfidence);
}
double aliasScore = linkset.Aliases.IsDefaultOrEmpty ? 0d : 1d;
double purlOverlapScore = linkset.PackageUrls.IsDefaultOrEmpty
? 0d
: (linkset.PackageUrls.Length > 1 ? 1d : 0.6d);
double cpeOverlapScore = linkset.Cpes.IsDefaultOrEmpty
? 0d
: (linkset.Cpes.Length > 1 ? 1d : 0.5d);
double severityAgreement = conflicts.Any(c => c.Reason == "severity-mismatch") ? 0.2d : 0.5d;
double referenceOverlap = linkset.References.IsDefaultOrEmpty ? 0d : 0.5d;
double freshnessScore = 0.5d; // until fetchedAt spread is available
var confidence = (0.40 * aliasScore) +
(0.25 * purlOverlapScore) +
(0.15 * cpeOverlapScore) +
(0.10 * severityAgreement) +
(0.05 * referenceOverlap) +
(0.05 * freshnessScore);
if (conflicts.Count > 0 && confidence > 0.7d)
{
confidence -= 0.1d; // penalize non-empty conflict sets
}
return Math.Clamp(confidence, 0d, 1d);
}
}

View File

@@ -0,0 +1,346 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Core.Linksets;
internal static class LinksetCorrelation
{
internal readonly record struct Input(
string? Vendor,
DateTimeOffset? FetchedAt,
IReadOnlyCollection<string> Aliases,
IReadOnlyCollection<string> Purls,
IReadOnlyCollection<string> Cpes,
IReadOnlyCollection<string> References);
internal static (double Confidence, IReadOnlyList<AdvisoryLinksetConflict> Conflicts) Compute(
IReadOnlyCollection<Input> inputs,
IReadOnlyList<AdvisoryLinksetConflict>? additionalConflicts = null)
{
if (inputs.Count == 0)
{
return (1.0, Array.Empty<AdvisoryLinksetConflict>());
}
var conflicts = new List<AdvisoryLinksetConflict>();
var aliasScore = CalculateAliasScore(inputs, conflicts);
var (purlScore, rangeConflicts) = CalculatePurlScore(inputs);
conflicts.AddRange(rangeConflicts);
var cpeScore = CalculateCpeScore(inputs);
var (referenceScore, referenceConflicts) = CalculateReferenceScore(inputs);
conflicts.AddRange(referenceConflicts);
var severityAgreement = 0.5d; // no severity data available in linkset inputs
var freshnessScore = CalculateFreshnessScore(inputs);
var baseConfidence = Clamp01(
(0.40d * aliasScore) +
(0.25d * purlScore) +
(0.15d * cpeScore) +
(0.10d * severityAgreement) +
(0.05d * referenceScore) +
(0.05d * freshnessScore));
if (conflicts.Count > 0 && baseConfidence > 0.7d)
{
baseConfidence -= 0.1d;
}
if (baseConfidence < 0.1d && conflicts.Count > 0)
{
baseConfidence = 0.1d; // keep deterministic low signal, not zero
}
if (additionalConflicts is { Count: > 0 })
{
conflicts.AddRange(additionalConflicts);
}
return (Clamp01(baseConfidence), DeduplicateAndSort(conflicts, inputs));
}
private static double CalculateAliasScore(IReadOnlyCollection<Input> inputs, List<AdvisoryLinksetConflict> conflicts)
{
if (inputs.Count == 1)
{
return inputs.First().Aliases.Count > 0 ? 1d : 0d;
}
var intersection = inputs
.Select(i => i.Aliases.Select(a => a.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal))
.Aggregate((acc, next) =>
{
acc.IntersectWith(next);
return acc;
});
if (intersection.Count > 0)
{
return 1d;
}
var anyAliases = inputs.Any(i => i.Aliases.Count > 0);
if (anyAliases)
{
var values = inputs
.Select(i => $"{i.Vendor ?? "source"}:{i.Aliases.FirstOrDefault() ?? "<none>"}")
.ToArray();
conflicts.Add(new AdvisoryLinksetConflict("aliases", "alias-inconsistency", values));
}
var vendors = inputs.Select(i => i.Vendor ?? string.Empty).ToHashSet(StringComparer.OrdinalIgnoreCase);
return vendors.Count == 1 ? 0.5d : 0d;
}
private static (double Score, IReadOnlyList<AdvisoryLinksetConflict> Conflicts) CalculatePurlScore(
IReadOnlyCollection<Input> inputs)
{
var conflicts = new List<AdvisoryLinksetConflict>();
if (inputs.All(i => i.Purls.Count == 0))
{
return (0d, conflicts);
}
List<HashSet<string>> packageKeysPerInput = inputs
.Select(i => i.Purls
.Select(ExtractPackageKey)
.Where(k => !string.IsNullOrEmpty(k))
.ToHashSet(StringComparer.Ordinal))
.ToList();
var sharedPackages = packageKeysPerInput
.Skip(1)
.Aggregate(
new HashSet<string>(packageKeysPerInput.First()!, StringComparer.Ordinal),
(acc, next) =>
{
acc.IntersectWith(next!);
return acc;
});
if (sharedPackages.Count > 0)
{
var hasExactPurlOverlap = HasExactPurlOverlap(inputs);
if (!hasExactPurlOverlap)
{
var divergent = CollectRangeConflicts(inputs, sharedPackages);
conflicts.AddRange(divergent);
}
return (hasExactPurlOverlap ? 1d : 0.6d, conflicts);
}
return (0d, conflicts);
}
private static IEnumerable<AdvisoryLinksetConflict> CollectRangeConflicts(
IReadOnlyCollection<Input> inputs,
HashSet<string> sharedPackages)
{
var conflicts = new List<AdvisoryLinksetConflict>();
foreach (var package in sharedPackages)
{
var values = inputs
.SelectMany(i => i.Purls
.Where(p => ExtractPackageKey(p) == package)
.Select(p => $"{i.Vendor ?? "source"}:{p}"))
.ToArray();
var sourceIds = inputs
.Select(i => i.Vendor ?? "source")
.ToArray();
if (values.Length > 1)
{
conflicts.Add(new AdvisoryLinksetConflict(
"affected.versions",
"affected-range-divergence",
values,
sourceIds));
}
}
return conflicts;
}
private static bool HasExactPurlOverlap(IReadOnlyCollection<Input> inputs)
{
var first = inputs.First().Purls.ToHashSet(StringComparer.Ordinal);
return inputs.Skip(1).Any(input => input.Purls.Any(first.Contains));
}
private static string ExtractPackageKey(string purl)
{
if (string.IsNullOrWhiteSpace(purl))
{
return string.Empty;
}
var atIndex = purl.LastIndexOf('@');
return atIndex > 0 ? purl[..atIndex] : purl;
}
private static double CalculateCpeScore(IReadOnlyCollection<Input> inputs)
{
if (inputs.All(i => i.Cpes.Count == 0))
{
return 0d;
}
var cpeSets = inputs.Select(i => i.Cpes.ToHashSet(StringComparer.OrdinalIgnoreCase)).ToList();
var exactOverlap = cpeSets.Skip(1).Any(set => set.Overlaps(cpeSets.First()));
if (exactOverlap)
{
return 1d;
}
var vendorProductSets = inputs
.Select(i => i.Cpes.Select(ParseVendorProduct).Where(vp => vp.vendor is not null).ToHashSet())
.ToList();
var sharedVendorProduct = vendorProductSets.Skip(1).Any(set => set.Overlaps(vendorProductSets.First()));
return sharedVendorProduct ? 0.5d : 0d;
}
private static (string? vendor, string? product) ParseVendorProduct(string cpe)
{
if (string.IsNullOrWhiteSpace(cpe))
{
return (null, null);
}
var parts = cpe.Split(':');
if (parts.Length >= 6 && parts[0].StartsWith("cpe", StringComparison.OrdinalIgnoreCase))
{
// cpe:2.3:a:vendor:product:version...
return (parts[3], parts[4]);
}
if (parts.Length >= 5 && parts[0] == "cpe" && parts[1] == "/")
{
return (parts[2], parts[3]);
}
return (null, null);
}
private static (double Score, IReadOnlyList<AdvisoryLinksetConflict> Conflicts) CalculateReferenceScore(
IReadOnlyCollection<Input> inputs)
{
var conflicts = new List<AdvisoryLinksetConflict>();
if (inputs.All(i => i.References.Count == 0))
{
return (0d, conflicts);
}
double maxOverlap = 0d;
var inputList = inputs.ToList();
for (var i = 0; i < inputList.Count; i++)
{
for (var j = i + 1; j < inputList.Count; j++)
{
var first = inputList[i].References.Select(r => r.ToLowerInvariant()).ToHashSet();
var second = inputList[j].References.Select(r => r.ToLowerInvariant()).ToHashSet();
var intersection = first.Intersect(second).Count();
var denom = Math.Max(first.Count, second.Count);
var overlap = denom == 0 ? 0d : (double)intersection / denom;
if (overlap > maxOverlap)
{
maxOverlap = overlap;
}
if (overlap == 0d && !string.Equals(inputList[i].Vendor, inputList[j].Vendor, StringComparison.OrdinalIgnoreCase))
{
var values = new[]
{
$"{inputList[i].Vendor ?? "source"}:{first.FirstOrDefault() ?? "<none>"}",
$"{inputList[j].Vendor ?? "source"}:{second.FirstOrDefault() ?? "<none>"}"
};
conflicts.Add(new AdvisoryLinksetConflict(
"references",
"reference-clash",
values,
new[]
{
inputList[i].Vendor ?? "source",
inputList[j].Vendor ?? "source"
}));
}
}
}
return (maxOverlap, conflicts);
}
private static double CalculateFreshnessScore(IReadOnlyCollection<Input> inputs)
{
var fetched = inputs
.Select(i => i.FetchedAt)
.Where(d => d.HasValue)
.Select(d => d!.Value)
.ToList();
if (fetched.Count <= 1)
{
return 0.5d; // neutral when unknown
}
var min = fetched.Min();
var max = fetched.Max();
var spread = max - min;
if (spread <= TimeSpan.FromHours(48))
{
return 1d;
}
if (spread >= TimeSpan.FromDays(14))
{
return 0d;
}
var remaining = TimeSpan.FromDays(14) - spread;
return Clamp01(remaining.TotalSeconds / TimeSpan.FromDays(14).TotalSeconds);
}
private static IReadOnlyList<AdvisoryLinksetConflict> DeduplicateAndSort(
IEnumerable<AdvisoryLinksetConflict> conflicts,
IReadOnlyCollection<Input> inputs)
{
var set = new HashSet<string>(StringComparer.Ordinal);
var list = new List<AdvisoryLinksetConflict>();
foreach (var conflict in conflicts)
{
var key = $"{conflict.Field}|{conflict.Reason}|{string.Join('|', conflict.Values ?? Array.Empty<string>())}";
if (set.Add(key))
{
if (conflict.SourceIds is null || conflict.SourceIds.Count == 0)
{
var allSources = inputs.Select(i => i.Vendor ?? "source").Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
list.Add(conflict with { SourceIds = allSources });
}
else
{
list.Add(conflict);
}
}
}
return list
.OrderBy(c => c.Field, StringComparer.Ordinal)
.ThenBy(c => c.Reason, StringComparer.Ordinal)
.ThenBy(c => string.Join('|', c.Values ?? Array.Empty<string>()), StringComparer.Ordinal)
.ToList();
}
private static double Clamp01(double value) => Math.Clamp(value, 0d, 1d);
}

View File

@@ -216,28 +216,27 @@ public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryS
var referenceSet = new HashSet<AdvisoryObservationReference>();
var scopeSet = new HashSet<string>(StringComparer.Ordinal);
var relationshipSet = new HashSet<RawRelationship>();
var conflictSet = new HashSet<string>(StringComparer.Ordinal);
var conflicts = new List<AdvisoryLinksetConflict>();
var confidence = 1.0;
var correlationInputs = new List<LinksetCorrelation.Input>(observations.Length);
foreach (var observation in observations)
{
foreach (var alias in observation.Linkset.Aliases)
{
aliasSet.Add(alias);
}
foreach (var purl in observation.Linkset.Purls)
{
purlSet.Add(purl);
}
foreach (var cpe in observation.Linkset.Cpes)
{
cpeSet.Add(cpe);
}
foreach (var reference in observation.Linkset.References)
aliasSet.Add(alias);
}
foreach (var purl in observation.Linkset.Purls)
{
purlSet.Add(purl);
}
foreach (var cpe in observation.Linkset.Cpes)
{
cpeSet.Add(cpe);
}
foreach (var reference in observation.Linkset.References)
{
referenceSet.Add(reference);
}
@@ -252,19 +251,17 @@ public sealed class AdvisoryObservationQueryService : IAdvisoryObservationQueryS
relationshipSet.Add(relationship);
}
var linksetProjection = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(observation.RawLinkset);
confidence = Math.Min(confidence, linksetProjection.confidence ?? 1.0);
foreach (var conflict in linksetProjection.conflicts)
{
var key = $"{conflict.Field}|{conflict.Reason}|{string.Join('|', conflict.Values ?? Array.Empty<string>())}";
if (conflictSet.Add(key))
{
conflicts.Add(conflict);
}
}
correlationInputs.Add(new LinksetCorrelation.Input(
observation.Source.Vendor,
observation.Upstream.FetchedAt,
observation.Linkset.Aliases,
observation.Linkset.Purls,
observation.Linkset.Cpes,
observation.Linkset.References.Select(r => r.Url).ToArray()));
}
var (confidence, conflicts) = LinksetCorrelation.Compute(correlationInputs);
return new AdvisoryObservationLinksetAggregate(
aliasSet.OrderBy(static alias => alias, StringComparer.Ordinal).ToImmutableArray(),
purlSet.OrderBy(static purl => purl, StringComparer.Ordinal).ToImmutableArray(),

View File

@@ -0,0 +1,12 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Observations;
/// <summary>
/// Transports advisory.observation.updated@1 events from the outbox to external subscribers (e.g., NATS).
/// </summary>
public interface IAdvisoryObservationEventTransport
{
Task SendAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken);
}

View File

@@ -108,5 +108,10 @@ public sealed class AdvisoryLinksetConflictDocument
[BsonElement("values")]
[BsonIgnoreIfNull]
public List<string>? Values { get; set; }
= null;
= new();
[BsonElement("sourceIds")]
[BsonIgnoreIfNull]
public List<string>? SourceIds { get; set; }
= new();
}

View File

@@ -111,7 +111,8 @@ internal sealed class ConcelierMongoLinksetStore : IMongoAdvisoryLinksetStore
{
Field = conflict.Field,
Reason = conflict.Reason,
Values = conflict.Values is null ? null : new List<string>(conflict.Values)
Values = conflict.Values is null ? null : new List<string>(conflict.Values),
SourceIds = conflict.SourceIds is null ? null : new List<string>(conflict.SourceIds)
}).ToList(),
Provenance = linkset.Provenance is null ? null : new AdvisoryLinksetProvenanceDocument
{
@@ -153,7 +154,8 @@ internal sealed class ConcelierMongoLinksetStore : IMongoAdvisoryLinksetStore
: doc.Conflicts.Select(conflict => new CoreLinksets.AdvisoryLinksetConflict(
conflict.Field,
conflict.Reason,
conflict.Values)).ToList(),
conflict.Values,
conflict.SourceIds)).ToList(),
DateTime.SpecifyKind(doc.CreatedAt, DateTimeKind.Utc),
doc.BuiltByJobId);
}

View File

@@ -0,0 +1,102 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Concelier.Storage.Mongo.Orchestrator;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
internal sealed class EnsureOrchestratorCollectionsMigration : IMongoMigration
{
public string Id => "20251122_orchestrator_registry_commands";
public string Description => "Ensure orchestrator registry, commands, and heartbeats collections exist with indexes";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
await EnsureRegistryAsync(database, cancellationToken).ConfigureAwait(false);
await EnsureCommandsAsync(database, cancellationToken).ConfigureAwait(false);
await EnsureHeartbeatsAsync(database, cancellationToken).ConfigureAwait(false);
}
private static async Task EnsureRegistryAsync(IMongoDatabase database, CancellationToken ct)
{
var name = MongoStorageDefaults.Collections.OrchestratorRegistry;
await EnsureCollectionAsync(database, name, ct).ConfigureAwait(false);
var collection = database.GetCollection<BsonDocument>(name);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
new(new BsonDocument
{
{"tenant", 1},
{"connectorId", 1},
}, new CreateIndexOptions { Name = "orch_registry_tenant_connector", Unique = true }),
new(new BsonDocument
{
{"source", 1},
}, new CreateIndexOptions { Name = "orch_registry_source" }),
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
private static async Task EnsureCommandsAsync(IMongoDatabase database, CancellationToken ct)
{
var name = MongoStorageDefaults.Collections.OrchestratorCommands;
await EnsureCollectionAsync(database, name, ct).ConfigureAwait(false);
var collection = database.GetCollection<BsonDocument>(name);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
new(new BsonDocument
{
{"tenant", 1},
{"connectorId", 1},
{"runId", 1},
{"sequence", 1},
}, new CreateIndexOptions { Name = "orch_cmd_tenant_connector_run_seq" }),
new(new BsonDocument { {"expiresAt", 1} }, new CreateIndexOptions
{
Name = "orch_cmd_expiresAt_ttl",
ExpireAfter = TimeSpan.FromSeconds(0),
})
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
private static async Task EnsureHeartbeatsAsync(IMongoDatabase database, CancellationToken ct)
{
var name = MongoStorageDefaults.Collections.OrchestratorHeartbeats;
await EnsureCollectionAsync(database, name, ct).ConfigureAwait(false);
var collection = database.GetCollection<BsonDocument>(name);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
new(new BsonDocument
{
{"tenant", 1},
{"connectorId", 1},
{"runId", 1},
{"sequence", 1},
}, new CreateIndexOptions { Name = "orch_hb_tenant_connector_run_seq" }),
new(new BsonDocument { {"timestamp", -1} }, new CreateIndexOptions { Name = "orch_hb_timestamp_desc" })
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
private static async Task EnsureCollectionAsync(IMongoDatabase database, string collectionName, CancellationToken ct)
{
var filter = new BsonDocument("name", collectionName);
using var cursor = await database.ListCollectionsAsync(new ListCollectionsOptions { Filter = filter }, ct).ConfigureAwait(false);
var exists = await cursor.AnyAsync(ct).ConfigureAwait(false);
if (!exists)
{
await database.CreateCollectionAsync(collectionName, cancellationToken: ct).ConfigureAwait(false);
}
}
}

View File

@@ -30,5 +30,8 @@ public static class MongoStorageDefaults
public const string AdvisoryObservations = "advisory_observations";
public const string AdvisoryLinksets = "advisory_linksets";
public const string AdvisoryObservationEvents = "advisory_observation_events";
public const string OrchestratorRegistry = "orchestrator_registry";
public const string OrchestratorCommands = "orchestrator_commands";
public const string OrchestratorHeartbeats = "orchestrator_heartbeats";
}
}

View File

@@ -11,18 +11,18 @@ namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class AdvisoryObservationTransportWorker : BackgroundService
{
private readonly IAdvisoryObservationEventOutbox _outbox;
private readonly IAdvisoryObservationEventPublisher _publisher;
private readonly IAdvisoryObservationEventTransport _transport;
private readonly ILogger<AdvisoryObservationTransportWorker> _logger;
private readonly AdvisoryObservationEventPublisherOptions _options;
public AdvisoryObservationTransportWorker(
IAdvisoryObservationEventOutbox outbox,
IAdvisoryObservationEventPublisher publisher,
IAdvisoryObservationEventTransport transport,
IOptions<AdvisoryObservationEventPublisherOptions> options,
ILogger<AdvisoryObservationTransportWorker> logger)
{
_outbox = outbox ?? throw new ArgumentNullException(nameof(outbox));
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = options.Value;
}
@@ -48,7 +48,7 @@ internal sealed class AdvisoryObservationTransportWorker : BackgroundService
foreach (var evt in batch)
{
await _publisher.PublishAsync(evt, stoppingToken).ConfigureAwait(false);
await _transport.SendAsync(evt, stoppingToken).ConfigureAwait(false);
await _outbox.MarkPublishedAsync(evt.EventId, DateTimeOffset.UtcNow, stoppingToken).ConfigureAwait(false);
}
}

View File

@@ -12,7 +12,7 @@ using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class NatsAdvisoryObservationEventPublisher : IAdvisoryObservationEventPublisher
internal sealed class NatsAdvisoryObservationEventPublisher : IAdvisoryObservationEventTransport
{
private readonly ILogger<NatsAdvisoryObservationEventPublisher> _logger;
private readonly AdvisoryObservationEventPublisherOptions _options;
@@ -26,7 +26,7 @@ internal sealed class NatsAdvisoryObservationEventPublisher : IAdvisoryObservati
_options = options.Value;
}
public async Task PublishAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken)
public async Task SendAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken)
{
if (!_options.Enabled)
{

View File

@@ -0,0 +1,15 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class NullAdvisoryObservationEventTransport : IAdvisoryObservationEventTransport
{
public static readonly NullAdvisoryObservationEventTransport Instance = new();
private NullAdvisoryObservationEventTransport() { }
public Task SendAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken)
=> Task.CompletedTask;
}

View File

@@ -0,0 +1,19 @@
namespace StellaOps.Concelier.Storage.Mongo.Orchestrator;
public interface IOrchestratorRegistryStore
{
Task UpsertAsync(OrchestratorRegistryRecord record, CancellationToken cancellationToken);
Task<OrchestratorRegistryRecord?> GetAsync(string tenant, string connectorId, CancellationToken cancellationToken);
Task EnqueueCommandAsync(OrchestratorCommandRecord command, CancellationToken cancellationToken);
Task<IReadOnlyList<OrchestratorCommandRecord>> GetPendingCommandsAsync(
string tenant,
string connectorId,
Guid runId,
long? afterSequence,
CancellationToken cancellationToken);
Task AppendHeartbeatAsync(OrchestratorHeartbeatRecord heartbeat, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,94 @@
using System;
using System.Linq;
using MongoDB.Driver;
namespace StellaOps.Concelier.Storage.Mongo.Orchestrator;
public sealed class MongoOrchestratorRegistryStore : IOrchestratorRegistryStore
{
private readonly IMongoCollection<OrchestratorRegistryDocument> _registry;
private readonly IMongoCollection<OrchestratorCommandDocument> _commands;
private readonly IMongoCollection<OrchestratorHeartbeatDocument> _heartbeats;
public MongoOrchestratorRegistryStore(
IMongoCollection<OrchestratorRegistryDocument> registry,
IMongoCollection<OrchestratorCommandDocument> commands,
IMongoCollection<OrchestratorHeartbeatDocument> heartbeats)
{
_registry = registry ?? throw new ArgumentNullException(nameof(registry));
_commands = commands ?? throw new ArgumentNullException(nameof(commands));
_heartbeats = heartbeats ?? throw new ArgumentNullException(nameof(heartbeats));
}
public async Task UpsertAsync(OrchestratorRegistryRecord record, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(record);
var document = OrchestratorRegistryDocumentExtensions.FromRecord(record);
var filter = Builders<OrchestratorRegistryDocument>.Filter.And(
Builders<OrchestratorRegistryDocument>.Filter.Eq(x => x.Tenant, record.Tenant),
Builders<OrchestratorRegistryDocument>.Filter.Eq(x => x.ConnectorId, record.ConnectorId));
var options = new ReplaceOptions { IsUpsert = true };
await _registry.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false);
}
public async Task<OrchestratorRegistryRecord?> GetAsync(string tenant, string connectorId, CancellationToken cancellationToken)
{
var filter = Builders<OrchestratorRegistryDocument>.Filter.And(
Builders<OrchestratorRegistryDocument>.Filter.Eq(x => x.Tenant, tenant),
Builders<OrchestratorRegistryDocument>.Filter.Eq(x => x.ConnectorId, connectorId));
var document = await _registry
.Find(filter)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return document?.ToRecord();
}
public async Task EnqueueCommandAsync(OrchestratorCommandRecord command, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(command);
var document = OrchestratorCommandDocumentExtensions.FromRecord(command);
await _commands.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
}
public async Task<IReadOnlyList<OrchestratorCommandRecord>> GetPendingCommandsAsync(
string tenant,
string connectorId,
Guid runId,
long? afterSequence,
CancellationToken cancellationToken)
{
var filter = Builders<OrchestratorCommandDocument>.Filter.And(
Builders<OrchestratorCommandDocument>.Filter.Eq(x => x.Tenant, tenant),
Builders<OrchestratorCommandDocument>.Filter.Eq(x => x.ConnectorId, connectorId),
Builders<OrchestratorCommandDocument>.Filter.Eq(x => x.RunId, runId));
if (afterSequence.HasValue)
{
filter &= Builders<OrchestratorCommandDocument>.Filter.Gt(x => x.Sequence, afterSequence.Value);
}
var results = await _commands
.Find(filter)
.Sort(Builders<OrchestratorCommandDocument>.Sort.Ascending(x => x.Sequence))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return results
.Select(static c => c.ToRecord())
.ToArray();
}
public async Task AppendHeartbeatAsync(OrchestratorHeartbeatRecord heartbeat, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(heartbeat);
var document = OrchestratorHeartbeatDocumentExtensions.FromRecord(heartbeat);
await _heartbeats.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,141 @@
using System;
using System.Globalization;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.Storage.Mongo.Orchestrator;
[BsonIgnoreExtraElements]
public sealed class OrchestratorCommandDocument
{
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("tenant")]
public string Tenant { get; set; } = string.Empty;
[BsonElement("connectorId")]
public string ConnectorId { get; set; } = string.Empty;
[BsonElement("runId")]
public Guid RunId { get; set; }
= Guid.Empty;
[BsonElement("sequence")]
public long Sequence { get; set; }
= 0;
[BsonElement("command")]
public OrchestratorCommandKind Command { get; set; }
= OrchestratorCommandKind.Pause;
[BsonElement("throttle")]
public OrchestratorThrottleOverrideDocument? Throttle { get; set; }
= null;
[BsonElement("backfill")]
public OrchestratorBackfillRangeDocument? Backfill { get; set; }
= null;
[BsonElement("createdAt")]
public DateTime CreatedAt { get; set; }
= DateTime.SpecifyKind(DateTime.UnixEpoch, DateTimeKind.Utc);
[BsonElement("expiresAt")]
public DateTime? ExpiresAt { get; set; }
= null;
}
[BsonIgnoreExtraElements]
public sealed class OrchestratorThrottleOverrideDocument
{
[BsonElement("rpm")]
public int? Rpm { get; set; }
= null;
[BsonElement("burst")]
public int? Burst { get; set; }
= null;
[BsonElement("cooldownSeconds")]
public int? CooldownSeconds { get; set; }
= null;
[BsonElement("expiresAt")]
public DateTime? ExpiresAt { get; set; }
= null;
}
[BsonIgnoreExtraElements]
public sealed class OrchestratorBackfillRangeDocument
{
[BsonElement("fromCursor")]
public string? FromCursor { get; set; }
= null;
[BsonElement("toCursor")]
public string? ToCursor { get; set; }
= null;
}
internal static class OrchestratorCommandDocumentExtensions
{
public static OrchestratorCommandDocument FromRecord(OrchestratorCommandRecord record)
{
ArgumentNullException.ThrowIfNull(record);
return new OrchestratorCommandDocument
{
Id = BuildId(record.Tenant, record.ConnectorId, record.RunId, record.Sequence),
Tenant = record.Tenant,
ConnectorId = record.ConnectorId,
RunId = record.RunId,
Sequence = record.Sequence,
Command = record.Command,
Throttle = record.Throttle is null
? null
: new OrchestratorThrottleOverrideDocument
{
Rpm = record.Throttle.Rpm,
Burst = record.Throttle.Burst,
CooldownSeconds = record.Throttle.CooldownSeconds,
ExpiresAt = record.Throttle.ExpiresAt?.UtcDateTime,
},
Backfill = record.Backfill is null
? null
: new OrchestratorBackfillRangeDocument
{
FromCursor = record.Backfill.FromCursor,
ToCursor = record.Backfill.ToCursor,
},
CreatedAt = record.CreatedAt.UtcDateTime,
ExpiresAt = record.ExpiresAt?.UtcDateTime,
};
}
public static OrchestratorCommandRecord ToRecord(this OrchestratorCommandDocument document)
{
ArgumentNullException.ThrowIfNull(document);
return new OrchestratorCommandRecord(
document.Tenant,
document.ConnectorId,
document.RunId,
document.Sequence,
document.Command,
document.Throttle is null
? null
: new OrchestratorThrottleOverride(
document.Throttle.Rpm,
document.Throttle.Burst,
document.Throttle.CooldownSeconds,
document.Throttle.ExpiresAt is null ? null : DateTime.SpecifyKind(document.Throttle.ExpiresAt.Value, DateTimeKind.Utc)),
document.Backfill is null
? null
: new OrchestratorBackfillRange(document.Backfill.FromCursor, document.Backfill.ToCursor),
DateTime.SpecifyKind(document.CreatedAt, DateTimeKind.Utc),
document.ExpiresAt is null ? null : DateTime.SpecifyKind(document.ExpiresAt.Value, DateTimeKind.Utc));
}
private static string BuildId(string tenant, string connectorId, Guid runId, long sequence)
=> string.Create(CultureInfo.InvariantCulture, $"{tenant}:{connectorId}:{runId}:{sequence}");
}

View File

@@ -0,0 +1,26 @@
using System;
namespace StellaOps.Concelier.Storage.Mongo.Orchestrator;
public sealed record OrchestratorCommandRecord(
string Tenant,
string ConnectorId,
Guid RunId,
long Sequence,
OrchestratorCommandKind Command,
OrchestratorThrottleOverride? Throttle,
OrchestratorBackfillRange? Backfill,
DateTimeOffset CreatedAt,
DateTimeOffset? ExpiresAt);
public enum OrchestratorCommandKind
{
Pause,
Resume,
Throttle,
Backfill,
}
public sealed record OrchestratorThrottleOverride(int? Rpm, int? Burst, int? CooldownSeconds, DateTimeOffset? ExpiresAt);
public sealed record OrchestratorBackfillRange(string? FromCursor, string? ToCursor);

View File

@@ -0,0 +1,105 @@
using System;
using System.Globalization;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.Storage.Mongo.Orchestrator;
[BsonIgnoreExtraElements]
public sealed class OrchestratorHeartbeatDocument
{
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("tenant")]
public string Tenant { get; set; } = string.Empty;
[BsonElement("connectorId")]
public string ConnectorId { get; set; } = string.Empty;
[BsonElement("runId")]
public Guid RunId { get; set; }
= Guid.Empty;
[BsonElement("sequence")]
public long Sequence { get; set; }
= 0;
[BsonElement("status")]
public OrchestratorHeartbeatStatus Status { get; set; }
= OrchestratorHeartbeatStatus.Starting;
[BsonElement("progress")]
public int? Progress { get; set; }
= null;
[BsonElement("queueDepth")]
public int? QueueDepth { get; set; }
= null;
[BsonElement("lastArtifactHash")]
public string? LastArtifactHash { get; set; }
= null;
[BsonElement("lastArtifactKind")]
public string? LastArtifactKind { get; set; }
= null;
[BsonElement("errorCode")]
public string? ErrorCode { get; set; }
= null;
[BsonElement("retryAfterSeconds")]
public int? RetryAfterSeconds { get; set; }
= null;
[BsonElement("timestamp")]
public DateTime Timestamp { get; set; }
= DateTime.SpecifyKind(DateTime.UnixEpoch, DateTimeKind.Utc);
}
internal static class OrchestratorHeartbeatDocumentExtensions
{
public static OrchestratorHeartbeatDocument FromRecord(OrchestratorHeartbeatRecord record)
{
ArgumentNullException.ThrowIfNull(record);
return new OrchestratorHeartbeatDocument
{
Id = BuildId(record.Tenant, record.ConnectorId, record.RunId, record.Sequence),
Tenant = record.Tenant,
ConnectorId = record.ConnectorId,
RunId = record.RunId,
Sequence = record.Sequence,
Status = record.Status,
Progress = record.Progress,
QueueDepth = record.QueueDepth,
LastArtifactHash = record.LastArtifactHash,
LastArtifactKind = record.LastArtifactKind,
ErrorCode = record.ErrorCode,
RetryAfterSeconds = record.RetryAfterSeconds,
Timestamp = record.TimestampUtc.UtcDateTime,
};
}
public static OrchestratorHeartbeatRecord ToRecord(this OrchestratorHeartbeatDocument document)
{
ArgumentNullException.ThrowIfNull(document);
return new OrchestratorHeartbeatRecord(
document.Tenant,
document.ConnectorId,
document.RunId,
document.Sequence,
document.Status,
document.Progress,
document.QueueDepth,
document.LastArtifactHash,
document.LastArtifactKind,
document.ErrorCode,
document.RetryAfterSeconds,
DateTime.SpecifyKind(document.Timestamp, DateTimeKind.Utc));
}
private static string BuildId(string tenant, string connectorId, Guid runId, long sequence)
=> string.Create(CultureInfo.InvariantCulture, $"{tenant}:{connectorId}:{runId}:{sequence}");
}

View File

@@ -0,0 +1,28 @@
using System;
namespace StellaOps.Concelier.Storage.Mongo.Orchestrator;
public sealed record OrchestratorHeartbeatRecord(
string Tenant,
string ConnectorId,
Guid RunId,
long Sequence,
OrchestratorHeartbeatStatus Status,
int? Progress,
int? QueueDepth,
string? LastArtifactHash,
string? LastArtifactKind,
string? ErrorCode,
int? RetryAfterSeconds,
DateTimeOffset TimestampUtc);
public enum OrchestratorHeartbeatStatus
{
Starting,
Running,
Paused,
Throttled,
Backfill,
Failed,
Succeeded,
}

View File

@@ -0,0 +1,165 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.Storage.Mongo.Orchestrator;
[BsonIgnoreExtraElements]
public sealed class OrchestratorRegistryDocument
{
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("tenant")]
public string Tenant { get; set; } = string.Empty;
[BsonElement("connectorId")]
public string ConnectorId { get; set; } = string.Empty;
[BsonElement("source")]
public string Source { get; set; } = string.Empty;
[BsonElement("capabilities")]
public IReadOnlyCollection<string> Capabilities { get; set; } = Array.Empty<string>();
[BsonElement("authRef")]
public string AuthRef { get; set; } = string.Empty;
[BsonElement("schedule")]
public OrchestratorScheduleDocument Schedule { get; set; } = new();
[BsonElement("ratePolicy")]
public OrchestratorRatePolicyDocument RatePolicy { get; set; } = new();
[BsonElement("artifactKinds")]
public IReadOnlyCollection<string> ArtifactKinds { get; set; } = Array.Empty<string>();
[BsonElement("lockKey")]
public string LockKey { get; set; } = string.Empty;
[BsonElement("egressGuard")]
public OrchestratorEgressGuardDocument EgressGuard { get; set; } = new();
[BsonElement("createdAt")]
public DateTime CreatedAt { get; set; }
= DateTime.SpecifyKind(DateTime.UnixEpoch, DateTimeKind.Utc);
[BsonElement("updatedAt")]
public DateTime UpdatedAt { get; set; }
= DateTime.SpecifyKind(DateTime.UnixEpoch, DateTimeKind.Utc);
}
[BsonIgnoreExtraElements]
public sealed class OrchestratorScheduleDocument
{
[BsonElement("cron")]
public string Cron { get; set; } = string.Empty;
[BsonElement("timeZone")]
public string TimeZone { get; set; } = "UTC";
[BsonElement("maxParallelRuns")]
public int MaxParallelRuns { get; set; }
= 1;
[BsonElement("maxLagMinutes")]
public int MaxLagMinutes { get; set; }
= 0;
}
[BsonIgnoreExtraElements]
public sealed class OrchestratorRatePolicyDocument
{
[BsonElement("rpm")]
public int Rpm { get; set; }
= 0;
[BsonElement("burst")]
public int Burst { get; set; }
= 0;
[BsonElement("cooldownSeconds")]
public int CooldownSeconds { get; set; }
= 0;
}
[BsonIgnoreExtraElements]
public sealed class OrchestratorEgressGuardDocument
{
[BsonElement("allowlist")]
public IReadOnlyCollection<string> Allowlist { get; set; } = Array.Empty<string>();
[BsonElement("airgapMode")]
public bool AirgapMode { get; set; }
= true;
}
internal static class OrchestratorRegistryDocumentExtensions
{
public static OrchestratorRegistryDocument FromRecord(OrchestratorRegistryRecord record)
{
ArgumentNullException.ThrowIfNull(record);
return new OrchestratorRegistryDocument
{
Id = BuildId(record.Tenant, record.ConnectorId),
Tenant = record.Tenant,
ConnectorId = record.ConnectorId,
Source = record.Source,
Capabilities = record.Capabilities,
AuthRef = record.AuthRef,
Schedule = new OrchestratorScheduleDocument
{
Cron = record.Schedule.Cron,
TimeZone = record.Schedule.TimeZone,
MaxParallelRuns = record.Schedule.MaxParallelRuns,
MaxLagMinutes = record.Schedule.MaxLagMinutes,
},
RatePolicy = new OrchestratorRatePolicyDocument
{
Rpm = record.RatePolicy.Rpm,
Burst = record.RatePolicy.Burst,
CooldownSeconds = record.RatePolicy.CooldownSeconds,
},
ArtifactKinds = record.ArtifactKinds,
LockKey = record.LockKey,
EgressGuard = new OrchestratorEgressGuardDocument
{
Allowlist = record.EgressGuard.Allowlist,
AirgapMode = record.EgressGuard.AirgapMode,
},
CreatedAt = record.CreatedAt.UtcDateTime,
UpdatedAt = record.UpdatedAt.UtcDateTime,
};
}
public static OrchestratorRegistryRecord ToRecord(this OrchestratorRegistryDocument document)
{
ArgumentNullException.ThrowIfNull(document);
return new OrchestratorRegistryRecord(
document.Tenant,
document.ConnectorId,
document.Source,
document.Capabilities,
document.AuthRef,
new OrchestratorSchedule(
document.Schedule.Cron,
document.Schedule.TimeZone,
document.Schedule.MaxParallelRuns,
document.Schedule.MaxLagMinutes),
new OrchestratorRatePolicy(
document.RatePolicy.Rpm,
document.RatePolicy.Burst,
document.RatePolicy.CooldownSeconds),
document.ArtifactKinds,
document.LockKey,
new OrchestratorEgressGuard(document.EgressGuard.Allowlist, document.EgressGuard.AirgapMode),
DateTime.SpecifyKind(document.CreatedAt, DateTimeKind.Utc),
DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc));
}
private static string BuildId(string tenant, string connectorId)
=> string.Create(CultureInfo.InvariantCulture, $"{tenant}:{connectorId}");
}

View File

@@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Concelier.Storage.Mongo.Orchestrator;
public sealed record OrchestratorRegistryRecord(
string Tenant,
string ConnectorId,
string Source,
IReadOnlyCollection<string> Capabilities,
string AuthRef,
OrchestratorSchedule Schedule,
OrchestratorRatePolicy RatePolicy,
IReadOnlyCollection<string> ArtifactKinds,
string LockKey,
OrchestratorEgressGuard EgressGuard,
DateTimeOffset CreatedAt,
DateTimeOffset UpdatedAt);
public sealed record OrchestratorSchedule(
string Cron,
string TimeZone,
int MaxParallelRuns,
int MaxLagMinutes);
public sealed record OrchestratorRatePolicy(
int Rpm,
int Burst,
int CooldownSeconds);
public sealed record OrchestratorEgressGuard(
IReadOnlyCollection<string> Allowlist,
bool AirgapMode);

View File

@@ -23,6 +23,7 @@ using StellaOps.Concelier.Storage.Mongo.Migrations;
using StellaOps.Concelier.Storage.Mongo.Observations;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Storage.Mongo.Linksets;
using StellaOps.Concelier.Storage.Mongo.Orchestrator;
namespace StellaOps.Concelier.Storage.Mongo;
@@ -81,17 +82,17 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IAdvisoryObservationLookup, AdvisoryObservationLookup>();
services.AddSingleton<IAdvisoryEventRepository, MongoAdvisoryEventRepository>();
services.AddSingleton<IAdvisoryEventLog, AdvisoryEventLog>();
services.AddSingleton<MongoAdvisoryObservationEventPublisher>();
services.AddSingleton<IAdvisoryObservationEventPublisher, MongoAdvisoryObservationEventPublisher>();
services.AddSingleton<NatsAdvisoryObservationEventPublisher>();
services.AddSingleton<IAdvisoryObservationEventPublisher>(sp =>
services.AddSingleton<IAdvisoryObservationEventTransport>(sp =>
{
var options = sp.GetRequiredService<IOptions<AdvisoryObservationEventPublisherOptions>>().Value;
if (string.Equals(options.Transport, "nats", StringComparison.OrdinalIgnoreCase))
if (options.Enabled && string.Equals(options.Transport, "nats", StringComparison.OrdinalIgnoreCase))
{
return sp.GetRequiredService<NatsAdvisoryObservationEventPublisher>();
}
return sp.GetRequiredService<MongoAdvisoryObservationEventPublisher>();
return NullAdvisoryObservationEventTransport.Instance;
});
services.AddSingleton<IAdvisoryObservationEventOutbox, MongoAdvisoryObservationEventOutbox>();
services.AddSingleton<IAdvisoryRawRepository, MongoAdvisoryRawRepository>();
@@ -129,6 +130,24 @@ public static class ServiceCollectionExtensions
return database.GetCollection<AdvisoryObservationEventDocument>(MongoStorageDefaults.Collections.AdvisoryObservationEvents);
});
services.AddSingleton<IMongoCollection<OrchestratorRegistryDocument>>(static sp =>
{
var database = sp.GetRequiredService<IMongoDatabase>();
return database.GetCollection<OrchestratorRegistryDocument>(MongoStorageDefaults.Collections.OrchestratorRegistry);
});
services.AddSingleton<IMongoCollection<OrchestratorCommandDocument>>(static sp =>
{
var database = sp.GetRequiredService<IMongoDatabase>();
return database.GetCollection<OrchestratorCommandDocument>(MongoStorageDefaults.Collections.OrchestratorCommands);
});
services.AddSingleton<IMongoCollection<OrchestratorHeartbeatDocument>>(static sp =>
{
var database = sp.GetRequiredService<IMongoDatabase>();
return database.GetCollection<OrchestratorHeartbeatDocument>(MongoStorageDefaults.Collections.OrchestratorHeartbeats);
});
services.AddSingleton<IMongoCollection<AdvisoryLinksetDocument>>(static sp =>
{
var database = sp.GetRequiredService<IMongoDatabase>();
@@ -136,6 +155,7 @@ public static class ServiceCollectionExtensions
});
services.AddHostedService<RawDocumentRetentionService>();
services.AddHostedService<AdvisoryObservationTransportWorker>();
services.AddSingleton<MongoMigrationRunner>();
services.AddSingleton<IMongoMigration, EnsureDocumentExpiryIndexesMigration>();
@@ -149,9 +169,12 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IMongoMigration, EnsureAdvisoryEventCollectionsMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryObservationEventCollectionMigration>();
services.AddSingleton<IMongoMigration, SemVerStyleBackfillMigration>();
services.AddSingleton<IMongoMigration, EnsureOrchestratorCollectionsMigration>();
services.AddSingleton<IOrchestratorRegistryStore, MongoOrchestratorRegistryStore>();
services.AddSingleton<IHostedService, AdvisoryObservationTransportWorker>();
return services;
}
}
}

View File

@@ -60,6 +60,44 @@ public sealed class AdvisoryObservationAggregationTests
Assert.Null(normalized); // no purls supplied
}
[Fact]
public void BuildAggregateLinkset_ComputesConflictsAndConfidenceFromObservations()
{
var obsA = CreateObservation(
"obs-a",
new RawLinkset
{
Aliases = ImmutableArray.Create("CVE-2025-0001"),
PackageUrls = ImmutableArray.Create("pkg:npm/foo@1.0.0"),
References = ImmutableArray.Create(new RawReference("advisory", "https://a.example/advisory"))
},
fetchedAt: DateTimeOffset.UtcNow.AddHours(-1),
vendor: "vendor-a");
var obsB = CreateObservation(
"obs-b",
new RawLinkset
{
Aliases = ImmutableArray.Create("GHSA-xxxx-xxxx"),
PackageUrls = ImmutableArray.Create("pkg:npm/foo@2.0.0"),
References = ImmutableArray.Create(new RawReference("advisory", "https://b.example/advisory"))
},
fetchedAt: DateTimeOffset.UtcNow,
vendor: "vendor-b");
var method = typeof(AdvisoryObservationQueryService).GetMethod(
"BuildAggregateLinkset",
BindingFlags.NonPublic | BindingFlags.Static)!;
var aggregate = (AdvisoryObservationLinksetAggregate)method.Invoke(
null,
new object?[] { ImmutableArray.Create(obsA, obsB) })!;
Assert.Contains(aggregate.Conflicts, c => c.Reason == "alias-inconsistency");
Assert.Contains(aggregate.Conflicts, c => c.Reason == "affected-range-divergence");
Assert.True(aggregate.Confidence is > 0.0 and < 1.0);
}
[Fact]
public void BuildAggregateLinkset_EmptyInputReturnsEmptyArrays()
{
@@ -75,13 +113,17 @@ public sealed class AdvisoryObservationAggregationTests
Assert.True(aggregate.Relationships.IsEmpty);
}
private static AdvisoryObservation CreateObservation(string id, RawLinkset rawLinkset)
private static AdvisoryObservation CreateObservation(
string id,
RawLinkset rawLinkset,
DateTimeOffset? fetchedAt = null,
string vendor = "vendor")
{
var source = new AdvisoryObservationSource("vendor", "stream", "api");
var source = new AdvisoryObservationSource(vendor, "stream", "api");
var upstream = new AdvisoryObservationUpstream(
"adv-id",
null,
DateTimeOffset.UtcNow,
(fetchedAt ?? DateTimeOffset.UtcNow),
DateTimeOffset.UtcNow,
"sha256:abc",
new AdvisoryObservationSignature(false, null, null, null));

View File

@@ -36,7 +36,7 @@ public sealed class ConcelierMongoLinksetStoreTests : IClassFixture<MongoIntegra
0.82,
new List<AdvisoryLinksetConflict>
{
new("severity", "disagree", new[] { "HIGH", "MEDIUM" })
new("severity", "disagree", new[] { "HIGH", "MEDIUM" }, new[] { "source-a", "source-b" })
},
DateTimeOffset.UtcNow,
"job-1");
@@ -54,6 +54,7 @@ public sealed class ConcelierMongoLinksetStoreTests : IClassFixture<MongoIntegra
Assert.Single(document.Conflicts!);
Assert.Equal("severity", document.Conflicts![0].Field);
Assert.Equal("disagree", document.Conflicts![0].Reason);
Assert.Equal(new[] { "source-a", "source-b" }, document.Conflicts![0].SourceIds);
}
[Fact]
@@ -72,7 +73,8 @@ public sealed class ConcelierMongoLinksetStoreTests : IClassFixture<MongoIntegra
{
Field = "references",
Reason = "mismatch",
Values = new List<string> { "url1", "url2" }
Values = new List<string> { "url1", "url2" },
SourceIds = new List<string> { "src-a", "src-b" }
}
},
CreatedAt = DateTime.UtcNow
@@ -90,6 +92,7 @@ public sealed class ConcelierMongoLinksetStoreTests : IClassFixture<MongoIntegra
Assert.NotNull(model.Conflicts);
Assert.Single(model.Conflicts!);
Assert.Equal("references", model.Conflicts![0].Field);
Assert.Equal(new[] { "src-a", "src-b" }, model.Conflicts![0].SourceIds);
}
[Fact]

View File

@@ -0,0 +1,100 @@
using System;
using System.Collections.Immutable;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Storage.Mongo.Observations;
using StellaOps.Concelier.Models.Observations;
using Xunit;
namespace StellaOps.Concelier.Storage.Mongo.Tests.Observations;
public class AdvisoryObservationTransportWorkerTests
{
[Fact]
public async Task Worker_publishes_outbox_entries_and_marks_published_once()
{
var evt = new AdvisoryObservationUpdatedEvent(
Guid.NewGuid(),
"tenant-1",
"obs-1",
"adv-1",
new Models.Observations.AdvisoryObservationSource("vendor", "stream", "api", "1.0.0"),
new AdvisoryObservationLinksetSummary(
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<AdvisoryObservationRelationshipSummary>.Empty),
"doc-sha",
"hash-1",
DateTimeOffset.UtcNow,
ReplayCursor: "cursor-1",
supersedesId: null,
traceId: "trace-1");
var outbox = new FakeOutbox(evt);
var transport = new FakeTransport();
var options = Options.Create(new AdvisoryObservationEventPublisherOptions
{
Enabled = true,
Transport = "nats",
Subject = "subject",
Stream = "stream",
NatsUrl = "nats://localhost:4222"
});
var worker = new AdvisoryObservationTransportWorker(outbox, transport, options, NullLogger<AdvisoryObservationTransportWorker>.Instance);
await worker.StartAsync(CancellationToken.None);
await Task.Delay(150, CancellationToken.None);
await worker.StopAsync(CancellationToken.None);
Assert.Equal(1, transport.Sent.Count);
Assert.Equal(evt.EventId, transport.Sent[0].EventId);
Assert.Equal(1, outbox.MarkedCount);
}
private sealed class FakeOutbox : IAdvisoryObservationEventOutbox
{
private readonly AdvisoryObservationUpdatedEvent _event;
private bool _dequeued;
public int MarkedCount { get; private set; }
public FakeOutbox(AdvisoryObservationUpdatedEvent @event)
{
_event = @event;
}
public Task<IReadOnlyCollection<AdvisoryObservationUpdatedEvent>> DequeueAsync(int take, CancellationToken cancellationToken)
{
if (_dequeued)
{
return Task.FromResult<IReadOnlyCollection<AdvisoryObservationUpdatedEvent>>(Array.Empty<AdvisoryObservationUpdatedEvent>());
}
_dequeued = true;
return Task.FromResult<IReadOnlyCollection<AdvisoryObservationUpdatedEvent>>(new[] { _event });
}
public Task MarkPublishedAsync(Guid eventId, DateTimeOffset publishedAt, CancellationToken cancellationToken)
{
MarkedCount++;
return Task.CompletedTask;
}
}
private sealed class FakeTransport : IAdvisoryObservationEventTransport
{
public List<AdvisoryObservationUpdatedEvent> Sent { get; } = new();
public Task SendAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken)
{
Sent.Add(@event);
return Task.CompletedTask;
}
}
}

View File

@@ -603,6 +603,33 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Assert.Equal("GHSA-2025-0001", evidence!.AdvisoryKey);
Assert.Equal(2, evidence.Records.Count);
Assert.All(evidence.Records, record => Assert.Equal("tenant-a", record.Tenant));
Assert.Null(evidence.Attestation);
}
[Fact]
public async Task AdvisoryEvidenceEndpoint_AttachesAttestationWhenBundleProvided()
{
await SeedAdvisoryRawDocumentsAsync(
CreateAdvisoryRawDocument("tenant-a", "vendor-x", "GHSA-2025-0003", "sha256:201", new BsonDocument("id", "GHSA-2025-0003:1")));
var repoRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "..", ".."));
var sampleDir = Path.Combine(repoRoot, "docs", "samples", "evidence-bundle");
var tarPath = Path.Combine(sampleDir, "evidence-bundle-m0.tar.gz");
var manifestPath = Path.Combine(sampleDir, "manifest.json");
var transparencyPath = Path.Combine(sampleDir, "transparency.json");
using var client = _factory.CreateClient();
var requestUri = $"/vuln/evidence/advisories/GHSA-2025-0003?tenant=tenant-a&bundlePath={Uri.EscapeDataString(tarPath)}&manifestPath={Uri.EscapeDataString(manifestPath)}&transparencyPath={Uri.EscapeDataString(transparencyPath)}&pipelineVersion=git:test-sha";
var response = await client.GetAsync(requestUri);
response.EnsureSuccessStatusCode();
var evidence = await response.Content.ReadFromJsonAsync<AdvisoryEvidenceResponse>();
Assert.NotNull(evidence);
Assert.NotNull(evidence!.Attestation);
Assert.Equal("evidence-bundle-m0", evidence.Attestation!.SubjectName);
Assert.Equal("git:test-sha", evidence.Attestation.PipelineVersion);
Assert.Equal(tarPath, evidence.Attestation.EvidenceBundlePath);
}
[Fact]