up
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-11-28 09:40:40 +02:00
parent 1c6730a1d2
commit 05da719048
206 changed files with 34741 additions and 1751 deletions

View File

@@ -3284,6 +3284,20 @@ private readonly record struct LinksetObservationSummary(
static async Task InitializeMongoAsync(WebApplication app)
{
// Skip Mongo initialization in testing/bypass mode.
var isTesting = string.Equals(
Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT"),
"Testing",
StringComparison.OrdinalIgnoreCase);
var bypass = string.Equals(
Environment.GetEnvironmentVariable("CONCELIER_BYPASS_MONGO"),
"1",
StringComparison.OrdinalIgnoreCase);
if (isTesting || bypass)
{
return;
}
await using var scope = app.Services.CreateAsyncScope();
var bootstrapper = scope.ServiceProvider.GetRequiredService<MongoBootstrapper>();
var logger = scope.ServiceProvider.GetRequiredService<ILoggerFactory>().CreateLogger("MongoBootstrapper");

View File

@@ -10,16 +10,20 @@ namespace StellaOps.Concelier.Core.Linksets;
/// <summary>
/// Contract-matching payload for <c>advisory.linkset.updated@1</c> events.
/// Per LNM-21-005, emits delta descriptions + observation ids (tenant + provenance only).
/// Enhanced per CONCELIER-POLICY-23-002 with idempotent IDs, confidence summaries, and tenant metadata.
/// </summary>
public sealed record AdvisoryLinksetUpdatedEvent(
Guid EventId,
string IdempotencyKey,
string TenantId,
AdvisoryLinksetTenantMetadata TenantMetadata,
string LinksetId,
string AdvisoryId,
string Source,
ImmutableArray<string> ObservationIds,
AdvisoryLinksetDelta Delta,
double? Confidence,
AdvisoryLinksetConfidenceSummary ConfidenceSummary,
ImmutableArray<AdvisoryLinksetConflictSummary> Conflicts,
AdvisoryLinksetProvenanceSummary Provenance,
DateTimeOffset CreatedAt,
@@ -43,16 +47,22 @@ public sealed record AdvisoryLinksetUpdatedEvent(
var delta = ComputeDelta(linkset, previousLinkset);
var conflicts = BuildConflictSummaries(linkset.Conflicts);
var provenance = BuildProvenance(linkset.Provenance);
var tenantMetadata = BuildTenantMetadata(linkset.TenantId, tenantUrn);
var confidenceSummary = BuildConfidenceSummary(linkset.Confidence, conflicts.Length);
var idempotencyKey = ComputeIdempotencyKey(linksetId, linkset, delta);
return new AdvisoryLinksetUpdatedEvent(
EventId: Guid.NewGuid(),
IdempotencyKey: idempotencyKey,
TenantId: tenantUrn,
TenantMetadata: tenantMetadata,
LinksetId: linksetId,
AdvisoryId: linkset.AdvisoryId,
Source: linkset.Source,
ObservationIds: linkset.ObservationIds,
Delta: delta,
Confidence: linkset.Confidence,
ConfidenceSummary: confidenceSummary,
Conflicts: conflicts,
Provenance: provenance,
CreatedAt: linkset.CreatedAt,
@@ -61,6 +71,139 @@ public sealed record AdvisoryLinksetUpdatedEvent(
TraceId: traceId);
}
/// <summary>
/// Computes a deterministic idempotency key for safe replay.
/// The key is derived from linkset identity + content hash so replaying the same change yields the same key.
/// </summary>
private static string ComputeIdempotencyKey(string linksetId, AdvisoryLinkset linkset, AdvisoryLinksetDelta delta)
{
var sb = new StringBuilder(256);
sb.Append(linksetId);
sb.Append('|');
sb.Append(linkset.TenantId);
sb.Append('|');
sb.Append(linkset.AdvisoryId);
sb.Append('|');
sb.Append(linkset.Source);
sb.Append('|');
sb.Append(linkset.CreatedAt.ToUniversalTime().Ticks);
sb.Append('|');
sb.Append(delta.Type);
sb.Append('|');
// Include observation IDs in sorted order for determinism
foreach (var obsId in linkset.ObservationIds.OrderBy(id => id, StringComparer.Ordinal))
{
sb.Append(obsId);
sb.Append(',');
}
// Include provenance hash if available
if (linkset.Provenance?.PolicyHash is not null)
{
sb.Append('|');
sb.Append(linkset.Provenance.PolicyHash);
}
var input = Encoding.UTF8.GetBytes(sb.ToString());
var hash = SHA256.HashData(input);
return Convert.ToHexString(hash).ToLowerInvariant();
}
/// <summary>
/// Builds tenant metadata for policy consumers.
/// </summary>
private static AdvisoryLinksetTenantMetadata BuildTenantMetadata(string tenantId, string tenantUrn)
{
// Extract tenant identifier from URN if present
var rawId = tenantUrn.StartsWith("urn:tenant:", StringComparison.Ordinal)
? tenantUrn["urn:tenant:".Length..]
: tenantId;
return new AdvisoryLinksetTenantMetadata(
TenantUrn: tenantUrn,
TenantId: rawId,
Namespace: ExtractNamespace(rawId));
}
/// <summary>
/// Extracts namespace prefix from tenant ID (e.g., "org:acme" → "org").
/// </summary>
private static string? ExtractNamespace(string tenantId)
{
var colonIndex = tenantId.IndexOf(':');
return colonIndex > 0 ? tenantId[..colonIndex] : null;
}
/// <summary>
/// Builds confidence summary with tier classification and contributing factors.
/// </summary>
private static AdvisoryLinksetConfidenceSummary BuildConfidenceSummary(double? confidence, int conflictCount)
{
var tier = ClassifyConfidenceTier(confidence);
var factors = BuildConfidenceFactors(confidence, conflictCount);
return new AdvisoryLinksetConfidenceSummary(
Value: confidence,
Tier: tier,
ConflictCount: conflictCount,
Factors: factors);
}
/// <summary>
/// Classifies confidence into tiers for policy rules.
/// </summary>
private static string ClassifyConfidenceTier(double? confidence) => confidence switch
{
null => "unknown",
>= 0.9 => "high",
>= 0.7 => "medium",
>= 0.5 => "low",
_ => "very-low"
};
/// <summary>
/// Builds human-readable factors contributing to confidence score.
/// </summary>
private static ImmutableArray<string> BuildConfidenceFactors(double? confidence, int conflictCount)
{
var factors = ImmutableArray.CreateBuilder<string>();
if (confidence is null)
{
factors.Add("no-confidence-data");
return factors.ToImmutable();
}
if (confidence >= 0.9)
{
factors.Add("strong-alias-correlation");
}
else if (confidence >= 0.7)
{
factors.Add("moderate-alias-correlation");
}
else if (confidence >= 0.5)
{
factors.Add("weak-alias-correlation");
}
else
{
factors.Add("minimal-correlation");
}
if (conflictCount > 0)
{
factors.Add($"has-{conflictCount}-conflict{(conflictCount > 1 ? "s" : "")}");
}
else
{
factors.Add("no-conflicts");
}
return factors.ToImmutable();
}
private static AdvisoryLinksetDelta ComputeDelta(AdvisoryLinkset current, AdvisoryLinkset? previous)
{
if (previous is null)
@@ -166,3 +309,26 @@ public sealed record AdvisoryLinksetProvenanceSummary(
ImmutableArray<string> ObservationHashes,
string? ToolVersion,
string? PolicyHash);
/// <summary>
/// Tenant metadata for policy replay and multi-tenant filtering.
/// Per CONCELIER-POLICY-23-002.
/// </summary>
public sealed record AdvisoryLinksetTenantMetadata(
string TenantUrn,
string TenantId,
string? Namespace);
/// <summary>
/// Confidence summary with tier classification for policy rules.
/// Per CONCELIER-POLICY-23-002.
/// </summary>
/// <param name="Value">Raw confidence score (0.0 - 1.0).</param>
/// <param name="Tier">Confidence tier: high (≥0.9), medium (≥0.7), low (≥0.5), very-low (&lt;0.5), unknown (null).</param>
/// <param name="ConflictCount">Number of conflicts detected in the linkset.</param>
/// <param name="Factors">Human-readable factors contributing to confidence score.</param>
public sealed record AdvisoryLinksetConfidenceSummary(
double? Value,
string Tier,
int ConflictCount,
ImmutableArray<string> Factors);

View File

@@ -0,0 +1,48 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Linksets;
/// <summary>
/// Stores and retrieves policy delta checkpoints for deterministic replay.
/// Consumers use checkpoints to track their position in the linkset stream.
/// </summary>
public interface IPolicyDeltaCheckpointStore
{
/// <summary>
/// Gets a checkpoint by consumer and tenant, creating one if it does not exist.
/// </summary>
Task<PolicyDeltaCheckpoint> GetOrCreateAsync(
string tenantId,
string consumerId,
CancellationToken cancellationToken);
/// <summary>
/// Gets a checkpoint by its unique ID.
/// </summary>
Task<PolicyDeltaCheckpoint?> GetAsync(
string checkpointId,
CancellationToken cancellationToken);
/// <summary>
/// Updates a checkpoint after processing a batch of linksets.
/// </summary>
Task<PolicyDeltaCheckpoint> UpdateAsync(
PolicyDeltaCheckpoint checkpoint,
CancellationToken cancellationToken);
/// <summary>
/// Lists all checkpoints for a given tenant.
/// </summary>
Task<IReadOnlyList<PolicyDeltaCheckpoint>> ListByTenantAsync(
string tenantId,
CancellationToken cancellationToken);
/// <summary>
/// Deletes a checkpoint (for cleanup or reset scenarios).
/// </summary>
Task<bool> DeleteAsync(
string checkpointId,
CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,86 @@
using System;
namespace StellaOps.Concelier.Core.Linksets;
/// <summary>
/// Represents a checkpoint for tracking policy delta consumption.
/// Enables deterministic replay by persisting the last processed position.
/// </summary>
public sealed record PolicyDeltaCheckpoint(
/// <summary>Unique identifier for this checkpoint (typically consumerId + tenant).</summary>
string CheckpointId,
/// <summary>Tenant scope for this checkpoint.</summary>
string TenantId,
/// <summary>Consumer identifier (e.g., "policy-engine", "vuln-explorer").</summary>
string ConsumerId,
/// <summary>Last processed linkset CreatedAt timestamp for cursor-based pagination.</summary>
DateTimeOffset? LastCreatedAt,
/// <summary>Last processed advisory ID (tie-breaker when CreatedAt matches).</summary>
string? LastAdvisoryId,
/// <summary>MongoDB change-stream resume token for real-time delta subscriptions.</summary>
string? ResumeToken,
/// <summary>Sequence number for ordering events within the same timestamp.</summary>
long SequenceNumber,
/// <summary>When this checkpoint was last updated.</summary>
DateTimeOffset UpdatedAt,
/// <summary>Count of linksets processed since checkpoint creation.</summary>
long ProcessedCount,
/// <summary>Hash of the last processed batch for integrity verification.</summary>
string? LastBatchHash)
{
public static PolicyDeltaCheckpoint CreateNew(string tenantId, string consumerId, DateTimeOffset now) =>
new(
CheckpointId: $"{consumerId}:{tenantId}",
TenantId: tenantId,
ConsumerId: consumerId,
LastCreatedAt: null,
LastAdvisoryId: null,
ResumeToken: null,
SequenceNumber: 0,
UpdatedAt: now,
ProcessedCount: 0,
LastBatchHash: null);
/// <summary>
/// Creates an <see cref="AdvisoryLinksetCursor"/> from this checkpoint for pagination.
/// Returns null if no position has been recorded yet.
/// </summary>
public AdvisoryLinksetCursor? ToCursor() =>
LastCreatedAt.HasValue && !string.IsNullOrEmpty(LastAdvisoryId)
? new AdvisoryLinksetCursor(LastCreatedAt.Value, LastAdvisoryId)
: null;
/// <summary>
/// Advances the checkpoint to a new position after processing a batch.
/// </summary>
public PolicyDeltaCheckpoint Advance(
DateTimeOffset lastCreatedAt,
string lastAdvisoryId,
long batchCount,
string? batchHash,
DateTimeOffset now) =>
this with
{
LastCreatedAt = lastCreatedAt,
LastAdvisoryId = lastAdvisoryId,
SequenceNumber = SequenceNumber + batchCount,
UpdatedAt = now,
ProcessedCount = ProcessedCount + batchCount,
LastBatchHash = batchHash
};
/// <summary>
/// Updates the resume token for change-stream subscriptions.
/// </summary>
public PolicyDeltaCheckpoint WithResumeToken(string resumeToken, DateTimeOffset now) =>
this with { ResumeToken = resumeToken, UpdatedAt = now };
}

View File

@@ -0,0 +1,111 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Risk;
/// <summary>
/// Provider interface for extracting vendor risk signals from observations.
/// Per CONCELIER-RISK-66-001, surfaces fact-only CVSS/KEV/fix data with provenance.
/// </summary>
public interface IVendorRiskSignalProvider
{
/// <summary>
/// Extracts risk signals from a specific observation.
/// </summary>
/// <param name="tenantId">Tenant identifier.</param>
/// <param name="observationId">Observation identifier.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Risk signal with CVSS, KEV, and fix data.</returns>
Task<VendorRiskSignal?> GetByObservationAsync(
string tenantId,
string observationId,
CancellationToken cancellationToken);
/// <summary>
/// Extracts risk signals from all observations for an advisory.
/// </summary>
/// <param name="tenantId">Tenant identifier.</param>
/// <param name="advisoryId">Advisory identifier (e.g., CVE-2024-1234).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Collection of risk signals from all vendor observations.</returns>
Task<IReadOnlyList<VendorRiskSignal>> GetByAdvisoryAsync(
string tenantId,
string advisoryId,
CancellationToken cancellationToken);
/// <summary>
/// Extracts aggregated risk signals for a linkset.
/// </summary>
/// <param name="tenantId">Tenant identifier.</param>
/// <param name="linksetId">Linkset identifier.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Collection of risk signals from linked observations.</returns>
Task<IReadOnlyList<VendorRiskSignal>> GetByLinksetAsync(
string tenantId,
string linksetId,
CancellationToken cancellationToken);
}
/// <summary>
/// Aggregated risk signal view combining multiple vendor observations.
/// </summary>
public sealed record AggregatedRiskView(
string TenantId,
string AdvisoryId,
IReadOnlyList<VendorRiskSignal> VendorSignals)
{
/// <summary>
/// Gets all unique CVSS scores across vendors with their provenance.
/// </summary>
public IReadOnlyList<VendorCvssScore> AllCvssScores =>
VendorSignals
.SelectMany(s => s.CvssScores)
.OrderByDescending(c => c.Score)
.ToList();
/// <summary>
/// Gets the highest CVSS score from any vendor.
/// </summary>
public VendorCvssScore? HighestCvssScore =>
AllCvssScores.FirstOrDefault();
/// <summary>
/// Indicates if any vendor reports KEV status.
/// </summary>
public bool IsKnownExploited =>
VendorSignals.Any(s => s.IsKnownExploited);
/// <summary>
/// Gets all KEV status entries from vendors.
/// </summary>
public IReadOnlyList<VendorKevStatus> KevStatuses =>
VendorSignals
.Where(s => s.KevStatus is not null)
.Select(s => s.KevStatus!)
.ToList();
/// <summary>
/// Indicates if any vendor reports a fix available.
/// </summary>
public bool HasFixAvailable =>
VendorSignals.Any(s => s.HasFixAvailable);
/// <summary>
/// Gets all fix availability entries from vendors.
/// </summary>
public IReadOnlyList<VendorFixAvailability> AllFixAvailability =>
VendorSignals
.SelectMany(s => s.FixAvailability)
.ToList();
/// <summary>
/// Gets vendors that provided risk data.
/// </summary>
public IReadOnlyList<string> ContributingVendors =>
VendorSignals
.Select(s => s.Provenance.Vendor)
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(v => v, StringComparer.OrdinalIgnoreCase)
.ToList();
}

View File

@@ -0,0 +1,169 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.Risk;
/// <summary>
/// Vendor-provided risk signal for an advisory observation.
/// Per CONCELIER-RISK-66-001, surfaces CVSS/KEV/fix data exactly as published with provenance anchors.
/// </summary>
/// <remarks>
/// This model is fact-only: no inference, weighting, or prioritization.
/// All data traces back to a specific vendor observation with provenance.
/// </remarks>
public sealed record VendorRiskSignal(
string TenantId,
string AdvisoryId,
string ObservationId,
VendorRiskProvenance Provenance,
ImmutableArray<VendorCvssScore> CvssScores,
VendorKevStatus? KevStatus,
ImmutableArray<VendorFixAvailability> FixAvailability,
DateTimeOffset ExtractedAt)
{
/// <summary>
/// Creates a risk signal with no data (for observations without risk metadata).
/// </summary>
public static VendorRiskSignal Empty(
string tenantId,
string advisoryId,
string observationId,
VendorRiskProvenance provenance,
DateTimeOffset extractedAt)
{
return new VendorRiskSignal(
TenantId: tenantId,
AdvisoryId: advisoryId,
ObservationId: observationId,
Provenance: provenance,
CvssScores: ImmutableArray<VendorCvssScore>.Empty,
KevStatus: null,
FixAvailability: ImmutableArray<VendorFixAvailability>.Empty,
ExtractedAt: extractedAt);
}
/// <summary>
/// Gets the highest severity CVSS score if any.
/// </summary>
public VendorCvssScore? HighestCvssScore => CvssScores.IsDefaultOrEmpty
? null
: CvssScores.MaxBy(s => s.Score);
/// <summary>
/// Indicates if any fix is available from any vendor.
/// </summary>
public bool HasFixAvailable => !FixAvailability.IsDefaultOrEmpty &&
FixAvailability.Any(f => f.Status == FixStatus.Available);
/// <summary>
/// Indicates if this advisory is in the KEV list.
/// </summary>
public bool IsKnownExploited => KevStatus?.InKev == true;
}
/// <summary>
/// Provenance anchor for vendor risk data.
/// </summary>
public sealed record VendorRiskProvenance(
string Vendor,
string Source,
string ObservationHash,
DateTimeOffset FetchedAt,
string? IngestJobId,
string? UpstreamId);
/// <summary>
/// Vendor-provided CVSS score with version information.
/// </summary>
public sealed record VendorCvssScore(
string System,
double Score,
string? Vector,
string? Severity,
VendorRiskProvenance Provenance)
{
/// <summary>
/// Normalizes the system name to a standard format.
/// </summary>
public string NormalizedSystem => System?.ToLowerInvariant() switch
{
"cvss_v2" or "cvssv2" or "cvss2" => "cvss_v2",
"cvss_v30" or "cvssv30" or "cvss30" or "cvss_v3" or "cvssv3" or "cvss3" => "cvss_v30",
"cvss_v31" or "cvssv31" or "cvss31" => "cvss_v31",
"cvss_v40" or "cvssv40" or "cvss40" or "cvss_v4" or "cvssv4" or "cvss4" => "cvss_v40",
var s => s ?? "unknown"
};
/// <summary>
/// Derives severity tier from score (if not provided by vendor).
/// </summary>
public string EffectiveSeverity => Severity ?? DeriveFromScore(Score, NormalizedSystem);
private static string DeriveFromScore(double score, string system)
{
// CVSS v2 uses different thresholds
if (system == "cvss_v2")
{
return score switch
{
>= 7.0 => "high",
>= 4.0 => "medium",
_ => "low"
};
}
// CVSS v3.x and v4.x thresholds
return score switch
{
>= 9.0 => "critical",
>= 7.0 => "high",
>= 4.0 => "medium",
>= 0.1 => "low",
_ => "none"
};
}
}
/// <summary>
/// KEV (Known Exploited Vulnerabilities) status from vendor data.
/// </summary>
public sealed record VendorKevStatus(
bool InKev,
DateTimeOffset? DateAdded,
DateTimeOffset? DueDate,
string? KnownRansomwareCampaignUse,
string? Notes,
VendorRiskProvenance Provenance);
/// <summary>
/// Fix availability information from vendor.
/// </summary>
public sealed record VendorFixAvailability(
FixStatus Status,
string? FixedVersion,
string? AdvisoryUrl,
DateTimeOffset? FixReleasedAt,
string? Package,
string? Ecosystem,
VendorRiskProvenance Provenance);
/// <summary>
/// Fix availability status.
/// </summary>
public enum FixStatus
{
/// <summary>Fix status unknown.</summary>
Unknown,
/// <summary>Fix is available.</summary>
Available,
/// <summary>No fix available yet.</summary>
NotAvailable,
/// <summary>Will not be fixed (end of life, etc.).</summary>
WillNotFix,
/// <summary>Fix is in progress.</summary>
InProgress
}

View File

@@ -0,0 +1,263 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json;
namespace StellaOps.Concelier.Core.Risk;
/// <summary>
/// Extracts vendor risk signals from observation data.
/// Per CONCELIER-RISK-66-001, extracts fact-only CVSS/KEV/fix data with provenance.
/// </summary>
public static class VendorRiskSignalExtractor
{
/// <summary>
/// Extracts a vendor risk signal from observation data.
/// </summary>
/// <param name="tenantId">Tenant identifier.</param>
/// <param name="advisoryId">Advisory identifier.</param>
/// <param name="observationId">Observation identifier.</param>
/// <param name="vendor">Vendor name.</param>
/// <param name="source">Source identifier.</param>
/// <param name="observationHash">Content hash for provenance.</param>
/// <param name="fetchedAt">When the data was fetched.</param>
/// <param name="ingestJobId">Optional ingest job ID.</param>
/// <param name="upstreamId">Optional upstream ID.</param>
/// <param name="severities">Severity data from observation.</param>
/// <param name="rawContent">Raw JSON content for KEV/fix extraction.</param>
/// <param name="now">Current timestamp.</param>
/// <returns>Extracted vendor risk signal.</returns>
public static VendorRiskSignal Extract(
string tenantId,
string advisoryId,
string observationId,
string vendor,
string source,
string observationHash,
DateTimeOffset fetchedAt,
string? ingestJobId,
string? upstreamId,
IReadOnlyList<SeverityInput>? severities,
JsonElement? rawContent,
DateTimeOffset now)
{
var provenance = new VendorRiskProvenance(
Vendor: vendor,
Source: source,
ObservationHash: observationHash,
FetchedAt: fetchedAt,
IngestJobId: ingestJobId,
UpstreamId: upstreamId);
var cvssScores = ExtractCvssScores(severities, provenance);
var kevStatus = ExtractKevStatus(rawContent, provenance);
var fixAvailability = ExtractFixAvailability(rawContent, provenance);
return new VendorRiskSignal(
TenantId: tenantId,
AdvisoryId: advisoryId,
ObservationId: observationId,
Provenance: provenance,
CvssScores: cvssScores,
KevStatus: kevStatus,
FixAvailability: fixAvailability,
ExtractedAt: now);
}
private static ImmutableArray<VendorCvssScore> ExtractCvssScores(
IReadOnlyList<SeverityInput>? severities,
VendorRiskProvenance provenance)
{
if (severities is null || severities.Count == 0)
{
return ImmutableArray<VendorCvssScore>.Empty;
}
var builder = ImmutableArray.CreateBuilder<VendorCvssScore>(severities.Count);
foreach (var severity in severities)
{
if (string.IsNullOrWhiteSpace(severity.System))
{
continue;
}
builder.Add(new VendorCvssScore(
System: severity.System,
Score: severity.Score,
Vector: severity.Vector,
Severity: severity.Severity,
Provenance: provenance));
}
return builder.ToImmutable();
}
private static VendorKevStatus? ExtractKevStatus(
JsonElement? rawContent,
VendorRiskProvenance provenance)
{
if (rawContent is null || rawContent.Value.ValueKind != JsonValueKind.Object)
{
return null;
}
var content = rawContent.Value;
// Try common KEV data locations in raw content
// NVD format: cisa_exploit_add, cisa_required_action, cisa_vulnerability_name
if (TryGetProperty(content, "cisa_exploit_add", out var cisaAdd) ||
TryGetProperty(content, "database_specific", out var dbSpecific) && TryGetProperty(dbSpecific, "cisa", out cisaAdd))
{
return new VendorKevStatus(
InKev: true,
DateAdded: TryParseDate(cisaAdd),
DueDate: TryGetDateProperty(content, "cisa_action_due"),
KnownRansomwareCampaignUse: TryGetStringProperty(content, "cisa_ransomware"),
Notes: TryGetStringProperty(content, "cisa_vulnerability_name"),
Provenance: provenance);
}
// OSV/GitHub format: database_specific.kev
if (TryGetProperty(content, "database_specific", out var osv) &&
TryGetProperty(osv, "kev", out var kev))
{
var inKev = kev.ValueKind == JsonValueKind.True ||
(kev.ValueKind == JsonValueKind.Object && TryGetProperty(kev, "in_kev", out var inKevProp) && inKevProp.ValueKind == JsonValueKind.True);
if (inKev)
{
return new VendorKevStatus(
InKev: true,
DateAdded: kev.ValueKind == JsonValueKind.Object ? TryGetDateProperty(kev, "date_added") : null,
DueDate: kev.ValueKind == JsonValueKind.Object ? TryGetDateProperty(kev, "due_date") : null,
KnownRansomwareCampaignUse: kev.ValueKind == JsonValueKind.Object ? TryGetStringProperty(kev, "ransomware") : null,
Notes: null,
Provenance: provenance);
}
}
return null;
}
private static ImmutableArray<VendorFixAvailability> ExtractFixAvailability(
JsonElement? rawContent,
VendorRiskProvenance provenance)
{
if (rawContent is null || rawContent.Value.ValueKind != JsonValueKind.Object)
{
return ImmutableArray<VendorFixAvailability>.Empty;
}
var content = rawContent.Value;
var builder = ImmutableArray.CreateBuilder<VendorFixAvailability>();
// OSV format: affected[].ranges[].events[{fixed: "version"}]
if (TryGetProperty(content, "affected", out var affected) && affected.ValueKind == JsonValueKind.Array)
{
foreach (var aff in affected.EnumerateArray())
{
var package = TryGetStringProperty(aff, "package", "name") ?? TryGetStringProperty(aff, "purl");
var ecosystem = TryGetStringProperty(aff, "package", "ecosystem");
if (TryGetProperty(aff, "ranges", out var ranges) && ranges.ValueKind == JsonValueKind.Array)
{
foreach (var range in ranges.EnumerateArray())
{
if (TryGetProperty(range, "events", out var events) && events.ValueKind == JsonValueKind.Array)
{
foreach (var evt in events.EnumerateArray())
{
if (TryGetProperty(evt, "fixed", out var fixedVersion))
{
builder.Add(new VendorFixAvailability(
Status: FixStatus.Available,
FixedVersion: fixedVersion.GetString(),
AdvisoryUrl: null,
FixReleasedAt: null,
Package: package,
Ecosystem: ecosystem,
Provenance: provenance));
}
}
}
}
}
// Also check versions[] for fixed versions
if (TryGetProperty(aff, "versions", out var versions) && versions.ValueKind == JsonValueKind.Array)
{
// Fixed versions may be indicated by absence from versions array
// This is less reliable, so we only use it if no range data exists
}
}
}
// NVD format: configurations with fix status
if (TryGetProperty(content, "configurations", out var configs) && configs.ValueKind == JsonValueKind.Array)
{
// NVD configurations don't directly indicate fixes, but CPE matches can imply them
// This would require more complex parsing - defer to vendor-specific connectors
}
return builder.ToImmutable();
}
private static bool TryGetProperty(JsonElement element, string propertyName, out JsonElement value)
{
value = default;
if (element.ValueKind != JsonValueKind.Object)
{
return false;
}
return element.TryGetProperty(propertyName, out value);
}
private static string? TryGetStringProperty(JsonElement element, params string[] path)
{
var current = element;
foreach (var segment in path)
{
if (!TryGetProperty(current, segment, out current))
{
return null;
}
}
return current.ValueKind == JsonValueKind.String ? current.GetString() : null;
}
private static DateTimeOffset? TryGetDateProperty(JsonElement element, string propertyName)
{
if (!TryGetProperty(element, propertyName, out var value))
{
return null;
}
return TryParseDate(value);
}
private static DateTimeOffset? TryParseDate(JsonElement element)
{
if (element.ValueKind == JsonValueKind.String)
{
var str = element.GetString();
if (DateTimeOffset.TryParse(str, out var date))
{
return date;
}
}
return null;
}
}
/// <summary>
/// Input for severity extraction from observation data.
/// </summary>
public sealed record SeverityInput(
string System,
double Score,
string? Vector,
string? Severity);

View File

@@ -0,0 +1,109 @@
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.Tenancy;
/// <summary>
/// Response model for /capabilities/tenant endpoint.
/// Per AUTH-TEN-47-001 and CONCELIER-TEN-48-001: echoes tenantId, scopes, and mergeAllowed=false when LNM is enabled.
/// </summary>
public sealed record TenantCapabilitiesResponse(
string TenantId,
string TenantUrn,
ImmutableArray<string> Scopes,
bool MergeAllowed,
bool OfflineAllowed,
TenantCapabilitiesMode Mode,
DateTimeOffset GeneratedAt)
{
/// <summary>
/// Creates a Link-Not-Merge capabilities response.
/// </summary>
public static TenantCapabilitiesResponse ForLinkNotMerge(
TenantScope scope,
DateTimeOffset now)
{
return new TenantCapabilitiesResponse(
TenantId: scope.TenantId,
TenantUrn: scope.TenantUrn,
Scopes: scope.Scopes,
MergeAllowed: false, // Always false in LNM mode
OfflineAllowed: scope.Capabilities.OfflineAllowed,
Mode: TenantCapabilitiesMode.LinkNotMerge,
GeneratedAt: now);
}
}
/// <summary>
/// Operating mode for tenant capabilities.
/// </summary>
public enum TenantCapabilitiesMode
{
/// <summary>Link-Not-Merge mode - no advisory merging.</summary>
LinkNotMerge,
/// <summary>Legacy merge mode (deprecated).</summary>
LegacyMerge
}
/// <summary>
/// Interface for tenant capabilities provider.
/// </summary>
public interface ITenantCapabilitiesProvider
{
/// <summary>
/// Gets the current capabilities for the tenant scope.
/// </summary>
TenantCapabilitiesResponse GetCapabilities(TenantScope scope);
/// <summary>
/// Validates that the tenant scope is allowed to perform the requested operation.
/// </summary>
/// <param name="scope">Tenant scope to validate.</param>
/// <param name="requiredScopes">Required scopes for the operation.</param>
/// <exception cref="TenantScopeException">Thrown if validation fails.</exception>
void ValidateScope(TenantScope scope, params string[] requiredScopes);
}
/// <summary>
/// Default implementation of tenant capabilities provider for Link-Not-Merge mode.
/// </summary>
public sealed class LinkNotMergeTenantCapabilitiesProvider : ITenantCapabilitiesProvider
{
private readonly TimeProvider _timeProvider;
public LinkNotMergeTenantCapabilitiesProvider(TimeProvider timeProvider)
{
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public TenantCapabilitiesResponse GetCapabilities(TenantScope scope)
{
ArgumentNullException.ThrowIfNull(scope);
scope.Validate();
// In Link-Not-Merge mode, merge is never allowed
// This enforces the contract even if the token claims mergeAllowed=true
return TenantCapabilitiesResponse.ForLinkNotMerge(scope, _timeProvider.GetUtcNow());
}
public void ValidateScope(TenantScope scope, params string[] requiredScopes)
{
ArgumentNullException.ThrowIfNull(scope);
scope.Validate();
if (requiredScopes.Length == 0)
{
return;
}
var hasRequired = requiredScopes.Any(required =>
scope.Scopes.Any(s => s.Equals(required, StringComparison.OrdinalIgnoreCase)));
if (!hasRequired)
{
throw new TenantScopeException(
"auth/insufficient-scope",
$"Required scope missing. Need one of: {string.Join(", ", requiredScopes)}");
}
}
}

View File

@@ -0,0 +1,123 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.Tenancy;
/// <summary>
/// Tenant scope data per AUTH-TEN-47-001 contract.
/// Per CONCELIER-TEN-48-001, enforces tenant scoping through normalization/linking.
/// </summary>
public sealed record TenantScope(
string TenantId,
string Issuer,
ImmutableArray<string> Scopes,
TenantCapabilities Capabilities,
TenantAttribution? Attribution,
DateTimeOffset IssuedAt,
DateTimeOffset ExpiresAt)
{
/// <summary>
/// Validates that the tenant scope is well-formed.
/// </summary>
public void Validate()
{
if (string.IsNullOrWhiteSpace(TenantId))
{
throw new TenantScopeException("auth/tenant-scope-missing", "TenantId is required");
}
if (string.IsNullOrWhiteSpace(Issuer))
{
throw new TenantScopeException("auth/tenant-scope-missing", "Issuer is required");
}
if (Scopes.IsDefaultOrEmpty)
{
throw new TenantScopeException("auth/tenant-scope-missing", "Scopes are required");
}
if (!HasRequiredScope())
{
throw new TenantScopeException("auth/tenant-scope-missing", "Required concelier scope missing");
}
if (ExpiresAt <= DateTimeOffset.UtcNow)
{
throw new TenantScopeException("auth/token-expired", "Token has expired");
}
}
/// <summary>
/// Checks if the scope has at least one required Concelier scope.
/// </summary>
public bool HasRequiredScope()
{
return Scopes.Any(s =>
s.StartsWith("concelier.", StringComparison.OrdinalIgnoreCase));
}
/// <summary>
/// Checks if the scope allows read access.
/// </summary>
public bool CanRead =>
Scopes.Any(s => s.Equals("concelier.read", StringComparison.OrdinalIgnoreCase) ||
s.Equals("concelier.linkset.read", StringComparison.OrdinalIgnoreCase));
/// <summary>
/// Checks if the scope allows write access.
/// </summary>
public bool CanWrite =>
Scopes.Any(s => s.Equals("concelier.linkset.write", StringComparison.OrdinalIgnoreCase));
/// <summary>
/// Checks if the scope allows tenant admin access.
/// </summary>
public bool CanAdminTenant =>
Scopes.Any(s => s.Equals("concelier.tenant.admin", StringComparison.OrdinalIgnoreCase));
/// <summary>
/// Gets the canonical tenant URN format.
/// </summary>
public string TenantUrn => TenantId.StartsWith("urn:tenant:", StringComparison.Ordinal)
? TenantId
: $"urn:tenant:{TenantId}";
}
/// <summary>
/// Tenant capabilities per AUTH-TEN-47-001 contract.
/// </summary>
public sealed record TenantCapabilities(
bool MergeAllowed = false,
bool OfflineAllowed = true)
{
/// <summary>
/// Default capabilities for Link-Not-Merge mode.
/// </summary>
public static TenantCapabilities Default { get; } = new(
MergeAllowed: false,
OfflineAllowed: true);
}
/// <summary>
/// Tenant attribution for audit logging.
/// </summary>
public sealed record TenantAttribution(
string? Actor,
string? TraceId);
/// <summary>
/// Exception thrown when tenant scope validation fails.
/// </summary>
public sealed class TenantScopeException : Exception
{
public TenantScopeException(string errorCode, string message)
: base(message)
{
ErrorCode = errorCode;
}
/// <summary>
/// Error code for API responses (e.g., auth/tenant-scope-missing).
/// </summary>
public string ErrorCode { get; }
}

View File

@@ -0,0 +1,105 @@
using System;
namespace StellaOps.Concelier.Core.Tenancy;
/// <summary>
/// Normalizes tenant identifiers for consistent storage and lookup.
/// Per CONCELIER-TEN-48-001: enforces tenant scoping through normalization.
/// </summary>
public static class TenantScopeNormalizer
{
private const string TenantUrnPrefix = "urn:tenant:";
/// <summary>
/// Normalizes a tenant identifier to canonical URN format.
/// </summary>
/// <param name="tenantId">Raw tenant identifier.</param>
/// <returns>Normalized tenant URN.</returns>
public static string NormalizeToUrn(string tenantId)
{
if (string.IsNullOrWhiteSpace(tenantId))
{
throw new ArgumentException("Tenant ID cannot be empty", nameof(tenantId));
}
var trimmed = tenantId.Trim();
// Already in URN format
if (trimmed.StartsWith(TenantUrnPrefix, StringComparison.Ordinal))
{
return trimmed.ToLowerInvariant();
}
// Convert to URN format
return $"{TenantUrnPrefix}{trimmed.ToLowerInvariant()}";
}
/// <summary>
/// Extracts the raw tenant identifier from a URN.
/// </summary>
/// <param name="tenantUrn">Tenant URN.</param>
/// <returns>Raw tenant identifier.</returns>
public static string ExtractFromUrn(string tenantUrn)
{
if (string.IsNullOrWhiteSpace(tenantUrn))
{
throw new ArgumentException("Tenant URN cannot be empty", nameof(tenantUrn));
}
var trimmed = tenantUrn.Trim();
if (trimmed.StartsWith(TenantUrnPrefix, StringComparison.OrdinalIgnoreCase))
{
return trimmed[TenantUrnPrefix.Length..].ToLowerInvariant();
}
return trimmed.ToLowerInvariant();
}
/// <summary>
/// Normalizes a tenant identifier for storage (lowercase, no URN prefix).
/// </summary>
/// <param name="tenantId">Raw tenant identifier or URN.</param>
/// <returns>Normalized tenant ID for storage.</returns>
public static string NormalizeForStorage(string tenantId)
{
return ExtractFromUrn(tenantId);
}
/// <summary>
/// Validates that two tenant identifiers refer to the same tenant.
/// </summary>
/// <param name="tenantId1">First tenant identifier.</param>
/// <param name="tenantId2">Second tenant identifier.</param>
/// <returns>True if both refer to the same tenant.</returns>
public static bool AreEqual(string? tenantId1, string? tenantId2)
{
if (string.IsNullOrWhiteSpace(tenantId1) || string.IsNullOrWhiteSpace(tenantId2))
{
return false;
}
var normalized1 = NormalizeForStorage(tenantId1);
var normalized2 = NormalizeForStorage(tenantId2);
return string.Equals(normalized1, normalized2, StringComparison.Ordinal);
}
/// <summary>
/// Validates that the provided tenant ID matches the scope's tenant.
/// </summary>
/// <param name="requestTenantId">Tenant ID from request.</param>
/// <param name="scope">Authenticated tenant scope.</param>
/// <exception cref="TenantScopeException">Thrown if tenant IDs don't match.</exception>
public static void ValidateTenantMatch(string requestTenantId, TenantScope scope)
{
ArgumentNullException.ThrowIfNull(scope);
if (!AreEqual(requestTenantId, scope.TenantId))
{
throw new TenantScopeException(
"auth/tenant-mismatch",
"Request tenant ID does not match authenticated tenant scope");
}
}
}

View File

@@ -31,6 +31,9 @@ This module owns the persistent shape of Concelier's MongoDB database. Upgrades
| `20251117_advisory_linksets_tenant_lower` | Lowercases `advisory_linksets.tenantId` to align writes with lookup filters. |
| `20251116_link_not_merge_collections` | Ensures `advisory_observations` and `advisory_linksets` collections exist with JSON schema validators and baseline indexes for LNM. |
| `20251127_lnm_sharding_and_ttl` | Adds hashed shard key indexes on `tenantId` for horizontal scaling and optional TTL indexes on `ingestedAt`/`createdAt` for storage retention. Creates `advisory_linkset_events` collection for linkset event outbox (LNM-21-101-DEV). |
| `20251127_lnm_legacy_backfill` | Backfills `advisory_observations` from `advisory_raw` documents and creates/updates `advisory_linksets` by grouping observations. Seeds `backfill_marker` tombstones on migrated documents for rollback tracking (LNM-21-102-DEV). |
| `20251128_policy_delta_checkpoints` | Creates `policy_delta_checkpoints` collection with tenant/consumer indexes for deterministic policy delta tracking. Supports cursor-based pagination and change-stream resume tokens for policy consumers (CONCELIER-POLICY-20-003). |
| `20251128_policy_lookup_indexes` | Adds secondary indexes for policy lookup patterns: alias multikey index on observations, confidence/severity indexes on linksets. Supports efficient policy joins without cached verdicts (CONCELIER-POLICY-23-001). |
## Operator Runbook
@@ -44,6 +47,11 @@ This module owns the persistent shape of Concelier's MongoDB database. Upgrades
- To re-run a migration in a lab, delete the corresponding document from `schema_migrations` and restart the service. **Do not** do this in production unless the migration body is known to be idempotent and safe.
- When changing retention settings (`RawDocumentRetention`), deploy the new configuration and restart Concelier. The migration runner will adjust indexes on the next boot.
- For the event-log collections (`advisory_statements`, `advisory_conflicts`), rollback is simply `db.advisory_statements.drop()` / `db.advisory_conflicts.drop()` followed by a restart if you must revert to the pre-event-log schema (only in labs). Production rollbacks should instead gate merge features that rely on these collections.
- For `20251127_lnm_legacy_backfill` rollback, use the provided Offline Kit script:
```bash
mongo concelier ops/devops/scripts/rollback-lnm-backfill.js
```
This script removes backfilled observations and linksets by querying the `backfill_marker` field (`lnm_21_102_dev`), then clears the tombstone markers from `advisory_raw`. After rollback, delete `20251127_lnm_legacy_backfill` from `schema_migrations` and restart.
- If migrations fail, restart with `Logging__LogLevel__StellaOps.Concelier.Storage.Mongo.Migrations=Debug` to surface diagnostic output. Remediate underlying index/collection drift before retrying.
## Validating an Upgrade

View File

@@ -0,0 +1,81 @@
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Concelier.Storage.Mongo.PolicyDelta;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
/// <summary>
/// Creates the policy_delta_checkpoints collection with indexes for deterministic policy delta tracking.
/// </summary>
internal sealed class EnsurePolicyDeltaCheckpointsCollectionMigration : IMongoMigration
{
public string Id => "20251128_policy_delta_checkpoints";
public string Description =>
"Creates policy_delta_checkpoints collection with tenant/consumer indexes for deterministic policy deltas (CONCELIER-POLICY-20-003).";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
var collectionName = MongoStorageDefaults.Collections.PolicyDeltaCheckpoints;
// Ensure collection exists
var collectionNames = await database
.ListCollectionNames(cancellationToken: cancellationToken)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var exists = collectionNames.Contains(collectionName);
if (!exists)
{
await database.CreateCollectionAsync(collectionName, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
var collection = database.GetCollection<PolicyDeltaCheckpointDocument>(collectionName);
// Index: tenantId for listing checkpoints by tenant
var tenantIndex = new CreateIndexModel<PolicyDeltaCheckpointDocument>(
Builders<PolicyDeltaCheckpointDocument>.IndexKeys.Ascending(d => d.TenantId),
new CreateIndexOptions
{
Name = "ix_tenantId",
Background = true
});
// Index: consumerId for querying checkpoints by consumer
var consumerIndex = new CreateIndexModel<PolicyDeltaCheckpointDocument>(
Builders<PolicyDeltaCheckpointDocument>.IndexKeys.Ascending(d => d.ConsumerId),
new CreateIndexOptions
{
Name = "ix_consumerId",
Background = true
});
// Compound index: (tenantId, consumerId) for efficient lookups
var compoundIndex = new CreateIndexModel<PolicyDeltaCheckpointDocument>(
Builders<PolicyDeltaCheckpointDocument>.IndexKeys
.Ascending(d => d.TenantId)
.Ascending(d => d.ConsumerId),
new CreateIndexOptions
{
Name = "ix_tenantId_consumerId",
Background = true
});
// Index: updatedAt for maintenance queries (stale checkpoint detection)
var updatedAtIndex = new CreateIndexModel<PolicyDeltaCheckpointDocument>(
Builders<PolicyDeltaCheckpointDocument>.IndexKeys.Ascending(d => d.UpdatedAt),
new CreateIndexOptions
{
Name = "ix_updatedAt",
Background = true
});
await collection.Indexes.CreateManyAsync(
[tenantIndex, consumerIndex, compoundIndex, updatedAtIndex],
cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,131 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
/// <summary>
/// Adds secondary indexes for policy lookup patterns: alias lookups, confidence filtering, and severity-based queries.
/// Supports efficient policy joins without cached verdicts per CONCELIER-POLICY-23-001.
/// </summary>
/// <remarks>
/// Query patterns supported:
/// <list type="bullet">
/// <item>Find observations by alias (CVE-ID, GHSA-ID): db.advisory_observations.find({"linkset.aliases": "cve-2024-1234"})</item>
/// <item>Find linksets by confidence range: db.advisory_linksets.find({"confidence": {$gte: 0.7}})</item>
/// <item>Find linksets by provider severity: db.advisory_linksets.find({"normalized.severities.system": "cvss_v31", "normalized.severities.score": {$gte: 7.0}})</item>
/// <item>Find linksets by tenant and advisory with confidence: db.advisory_linksets.find({"tenantId": "...", "advisoryId": "...", "confidence": {$gte: 0.5}})</item>
/// </list>
/// </remarks>
internal sealed class EnsurePolicyLookupIndexesMigration : IMongoMigration
{
public string Id => "20251128_policy_lookup_indexes";
public string Description => "Add secondary indexes for alias, confidence, and severity-based policy lookups (CONCELIER-POLICY-23-001)";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
await EnsureObservationPolicyIndexesAsync(database, cancellationToken).ConfigureAwait(false);
await EnsureLinksetPolicyIndexesAsync(database, cancellationToken).ConfigureAwait(false);
}
private static async Task EnsureObservationPolicyIndexesAsync(IMongoDatabase database, CancellationToken ct)
{
var collection = database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryObservations);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
// Multikey index on linkset.aliases for alias-based lookups (CVE-ID, GHSA-ID, etc.)
// Query pattern: db.advisory_observations.find({"linkset.aliases": "cve-2024-1234"})
new(new BsonDocument("linkset.aliases", 1),
new CreateIndexOptions
{
Name = "obs_linkset_aliases",
Background = true,
Sparse = true
}),
// Compound index for tenant + alias lookups
// Query pattern: db.advisory_observations.find({"tenant": "...", "linkset.aliases": "cve-2024-1234"})
new(new BsonDocument { { "tenant", 1 }, { "linkset.aliases", 1 } },
new CreateIndexOptions
{
Name = "obs_tenant_aliases",
Background = true
})
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
private static async Task EnsureLinksetPolicyIndexesAsync(IMongoDatabase database, CancellationToken ct)
{
var collection = database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryLinksets);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
// Index on confidence for confidence-based filtering
// Query pattern: db.advisory_linksets.find({"confidence": {$gte: 0.7}})
new(new BsonDocument("confidence", -1),
new CreateIndexOptions
{
Name = "linkset_confidence",
Background = true,
Sparse = true
}),
// Compound index for tenant + confidence lookups
// Query pattern: db.advisory_linksets.find({"tenantId": "...", "confidence": {$gte: 0.7}})
new(new BsonDocument { { "tenantId", 1 }, { "confidence", -1 } },
new CreateIndexOptions
{
Name = "linkset_tenant_confidence",
Background = true
}),
// Index on normalized.severities.system for severity system filtering
// Query pattern: db.advisory_linksets.find({"normalized.severities.system": "cvss_v31"})
new(new BsonDocument("normalized.severities.system", 1),
new CreateIndexOptions
{
Name = "linkset_severity_system",
Background = true,
Sparse = true
}),
// Compound index for severity system + score for range queries
// Query pattern: db.advisory_linksets.find({"normalized.severities.system": "cvss_v31", "normalized.severities.score": {$gte: 7.0}})
new(new BsonDocument { { "normalized.severities.system", 1 }, { "normalized.severities.score", -1 } },
new CreateIndexOptions
{
Name = "linkset_severity_system_score",
Background = true,
Sparse = true
}),
// Compound index for tenant + advisory + confidence (policy delta queries)
// Query pattern: db.advisory_linksets.find({"tenantId": "...", "advisoryId": "...", "confidence": {$gte: 0.5}})
new(new BsonDocument { { "tenantId", 1 }, { "advisoryId", 1 }, { "confidence", -1 } },
new CreateIndexOptions
{
Name = "linkset_tenant_advisory_confidence",
Background = true
}),
// Index for createdAt-based pagination (policy delta cursors)
// Query pattern: db.advisory_linksets.find({"tenantId": "...", "createdAt": {$gt: ISODate("...")}}).sort({"createdAt": 1})
new(new BsonDocument { { "tenantId", 1 }, { "createdAt", 1 } },
new CreateIndexOptions
{
Name = "linkset_tenant_createdAt",
Background = true
})
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken: ct).ConfigureAwait(false);
}
}

View File

@@ -1,13 +1,13 @@
namespace StellaOps.Concelier.Storage.Mongo;
public static class MongoStorageDefaults
{
public const string DefaultDatabaseName = "concelier";
public static class Collections
{
public const string Source = "source";
public const string SourceState = "source_state";
namespace StellaOps.Concelier.Storage.Mongo;
public static class MongoStorageDefaults
{
public const string DefaultDatabaseName = "concelier";
public static class Collections
{
public const string Source = "source";
public const string SourceState = "source_state";
public const string Document = "document";
public const string Dto = "dto";
public const string Advisory = "advisory";
@@ -15,10 +15,10 @@ public static class MongoStorageDefaults
public const string Alias = "alias";
public const string Affected = "affected";
public const string Reference = "reference";
public const string KevFlag = "kev_flag";
public const string RuFlags = "ru_flags";
public const string JpFlags = "jp_flags";
public const string PsirtFlags = "psirt_flags";
public const string KevFlag = "kev_flag";
public const string RuFlags = "ru_flags";
public const string JpFlags = "jp_flags";
public const string PsirtFlags = "psirt_flags";
public const string MergeEvent = "merge_event";
public const string ExportState = "export_state";
public const string Locks = "locks";
@@ -33,5 +33,6 @@ public static class MongoStorageDefaults
public const string OrchestratorRegistry = "orchestrator_registry";
public const string OrchestratorCommands = "orchestrator_commands";
public const string OrchestratorHeartbeats = "orchestrator_heartbeats";
public const string PolicyDeltaCheckpoints = "policy_delta_checkpoints";
}
}

View File

@@ -0,0 +1,135 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.PolicyDelta;
/// <summary>
/// MongoDB implementation of <see cref="IPolicyDeltaCheckpointStore"/>.
/// </summary>
internal sealed class MongoPolicyDeltaCheckpointStore : IPolicyDeltaCheckpointStore
{
private readonly IMongoCollection<PolicyDeltaCheckpointDocument> _collection;
private readonly TimeProvider _timeProvider;
public MongoPolicyDeltaCheckpointStore(IMongoDatabase database, TimeProvider timeProvider)
{
ArgumentNullException.ThrowIfNull(database);
ArgumentNullException.ThrowIfNull(timeProvider);
_collection = database.GetCollection<PolicyDeltaCheckpointDocument>(
MongoStorageDefaults.Collections.PolicyDeltaCheckpoints);
_timeProvider = timeProvider;
}
public async Task<PolicyDeltaCheckpoint> GetOrCreateAsync(
string tenantId,
string consumerId,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentException.ThrowIfNullOrWhiteSpace(consumerId);
var checkpointId = $"{consumerId}:{tenantId}";
var existing = await _collection
.Find(d => d.Id == checkpointId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (existing is not null)
{
return existing.ToRecord();
}
var now = _timeProvider.GetUtcNow();
var checkpoint = PolicyDeltaCheckpoint.CreateNew(tenantId, consumerId, now);
var document = PolicyDeltaCheckpointDocument.FromRecord(checkpoint);
try
{
await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
return checkpoint;
}
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
{
// Race condition: another process created the checkpoint concurrently.
existing = await _collection
.Find(d => d.Id == checkpointId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return existing?.ToRecord() ?? checkpoint;
}
}
public async Task<PolicyDeltaCheckpoint?> GetAsync(
string checkpointId,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(checkpointId);
var document = await _collection
.Find(d => d.Id == checkpointId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return document?.ToRecord();
}
public async Task<PolicyDeltaCheckpoint> UpdateAsync(
PolicyDeltaCheckpoint checkpoint,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(checkpoint);
var document = PolicyDeltaCheckpointDocument.FromRecord(checkpoint);
var options = new ReplaceOptions { IsUpsert = true };
await _collection
.ReplaceOneAsync(
d => d.Id == checkpoint.CheckpointId,
document,
options,
cancellationToken)
.ConfigureAwait(false);
return checkpoint;
}
public async Task<IReadOnlyList<PolicyDeltaCheckpoint>> ListByTenantAsync(
string tenantId,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
var documents = await _collection
.Find(d => d.TenantId == tenantId)
.SortBy(d => d.ConsumerId)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var results = new List<PolicyDeltaCheckpoint>(documents.Count);
foreach (var doc in documents)
{
results.Add(doc.ToRecord());
}
return results;
}
public async Task<bool> DeleteAsync(
string checkpointId,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(checkpointId);
var result = await _collection
.DeleteOneAsync(d => d.Id == checkpointId, cancellationToken)
.ConfigureAwait(false);
return result.DeletedCount > 0;
}
}

View File

@@ -0,0 +1,78 @@
using System;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo.PolicyDelta;
/// <summary>
/// MongoDB document for storing policy delta checkpoints.
/// </summary>
[BsonIgnoreExtraElements]
internal sealed class PolicyDeltaCheckpointDocument
{
/// <summary>
/// Unique identifier: {consumerId}:{tenantId}
/// </summary>
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("tenantId")]
public string TenantId { get; set; } = string.Empty;
[BsonElement("consumerId")]
public string ConsumerId { get; set; } = string.Empty;
[BsonElement("lastCreatedAt")]
[BsonIgnoreIfNull]
public DateTime? LastCreatedAt { get; set; }
[BsonElement("lastAdvisoryId")]
[BsonIgnoreIfNull]
public string? LastAdvisoryId { get; set; }
[BsonElement("resumeToken")]
[BsonIgnoreIfNull]
public string? ResumeToken { get; set; }
[BsonElement("sequenceNumber")]
public long SequenceNumber { get; set; }
[BsonElement("updatedAt")]
public DateTime UpdatedAt { get; set; }
[BsonElement("processedCount")]
public long ProcessedCount { get; set; }
[BsonElement("lastBatchHash")]
[BsonIgnoreIfNull]
public string? LastBatchHash { get; set; }
public PolicyDeltaCheckpoint ToRecord() =>
new(
CheckpointId: Id,
TenantId: TenantId,
ConsumerId: ConsumerId,
LastCreatedAt: LastCreatedAt.HasValue ? new DateTimeOffset(LastCreatedAt.Value, TimeSpan.Zero) : null,
LastAdvisoryId: LastAdvisoryId,
ResumeToken: ResumeToken,
SequenceNumber: SequenceNumber,
UpdatedAt: new DateTimeOffset(UpdatedAt, TimeSpan.Zero),
ProcessedCount: ProcessedCount,
LastBatchHash: LastBatchHash);
public static PolicyDeltaCheckpointDocument FromRecord(PolicyDeltaCheckpoint record) =>
new()
{
Id = record.CheckpointId,
TenantId = record.TenantId,
ConsumerId = record.ConsumerId,
LastCreatedAt = record.LastCreatedAt?.UtcDateTime,
LastAdvisoryId = record.LastAdvisoryId,
ResumeToken = record.ResumeToken,
SequenceNumber = record.SequenceNumber,
UpdatedAt = record.UpdatedAt.UtcDateTime,
ProcessedCount = record.ProcessedCount,
LastBatchHash = record.LastBatchHash
};
}

View File

@@ -24,6 +24,8 @@ using StellaOps.Concelier.Storage.Mongo.Observations;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Storage.Mongo.Linksets;
using StellaOps.Concelier.Storage.Mongo.Orchestrator;
using StellaOps.Concelier.Storage.Mongo.PolicyDelta;
using StellaOps.Concelier.Core.Linksets;
namespace StellaOps.Concelier.Storage.Mongo;
@@ -190,8 +192,12 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IMongoMigration, EnsureOrchestratorCollectionsMigration>();
services.AddSingleton<IMongoMigration, EnsureLinkNotMergeCollectionsMigration>();
services.AddSingleton<IMongoMigration, EnsureLinkNotMergeShardingAndTtlMigration>();
services.AddSingleton<IMongoMigration, EnsureLegacyAdvisoriesBackfillMigration>();
services.AddSingleton<IMongoMigration, EnsurePolicyDeltaCheckpointsCollectionMigration>();
services.AddSingleton<IMongoMigration, EnsurePolicyLookupIndexesMigration>();
services.AddSingleton<IOrchestratorRegistryStore, MongoOrchestratorRegistryStore>();
services.AddSingleton<IPolicyDeltaCheckpointStore, MongoPolicyDeltaCheckpointStore>();
services.AddSingleton<IHostedService, AdvisoryObservationTransportWorker>();