stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 16 Releases Wiki Activity

feat: Enhance MongoDB storage with event publishing and outbox support

Browse Source 
- Added `MongoAdvisoryObservationEventPublisher` and `NatsAdvisoryObservationEventPublisher` for event publishing.
- Registered `IAdvisoryObservationEventPublisher` to choose between NATS and MongoDB based on configuration.
- Introduced `MongoAdvisoryObservationEventOutbox` for outbox pattern implementation.
- Updated service collection to include new event publishers and outbox.
- Added a new hosted service `AdvisoryObservationTransportWorker` for processing events.

feat: Update project dependencies

- Added `NATS.Client.Core` package to the project for NATS integration.

test: Add unit tests for AdvisoryLinkset normalization

- Created `AdvisoryLinksetNormalizationConfidenceTests` to validate confidence score calculations.

fix: Adjust confidence assertion in `AdvisoryObservationAggregationTests`

- Updated confidence assertion to allow a range instead of a fixed value.

test: Implement tests for AdvisoryObservationEventFactory

- Added `AdvisoryObservationEventFactoryTests` to ensure correct mapping and hashing of observation events.

chore: Configure test project for Findings Ledger

- Created `Directory.Build.props` for test project configuration.
- Added `StellaOps.Findings.Ledger.Exports.Unit.csproj` for unit tests related to findings ledger exports.

feat: Implement export contracts for findings ledger

- Defined export request and response contracts in `ExportContracts.cs`.
- Created various export item records for findings, VEX, advisories, and SBOMs.

feat: Add export functionality to Findings Ledger Web Service

- Implemented endpoints for exporting findings, VEX, advisories, and SBOMs.
- Integrated `ExportQueryService` for handling export logic and pagination.

test: Add tests for Node language analyzer phase 22

- Implemented `NodePhase22SampleLoaderTests` to validate loading of NDJSON fixtures.
- Created sample NDJSON file for testing.

chore: Set up isolated test environment for Node tests

- Added `node-isolated.runsettings` for isolated test execution.
- Created `node-tests-isolated.sh` script for running tests in isolation.
This commit is contained in:
master
2025-11-20 23:08:45 +02:00
parent f0e74d2ee8
commit 2e276d6676
49 changed files with 1996 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/Concelier/StellaOps.Concelier.WebService/Program.cs
View File

@@ -106,6 +106,15 @@ builder.Services.AddMongoStorage(storageOptions =>
storageOptions.DatabaseName = concelierOptions.Storage.Database;
storageOptions.CommandTimeout = TimeSpan.FromSeconds(concelierOptions.Storage.CommandTimeoutSeconds);
});
builder.Services.AddOptions<AdvisoryObservationEventPublisherOptions>()
.Bind(builder.Configuration.GetSection("advisoryObservationEvents"))
.PostConfigure(options =>
{
options.Subject ??= "concelier.advisory.observation.updated.v1";
options.Stream ??= "CONCELIER_OBS";
options.Transport = string.IsNullOrWhiteSpace(options.Transport) ? "mongo" : options.Transport;
})
.ValidateOnStart();
builder.Services.AddConcelierAocGuards();
builder.Services.AddConcelierLinksetMappers();
builder.Services.AddAdvisoryRawServices();

46
src/Concelier/__Libraries/StellaOps.Concelier.Core/Linksets/AdvisoryLinksetNormalization.cs
View File

@@ -33,7 +33,7 @@ internal static class AdvisoryLinksetNormalization
var normalized = Build(linkset.PackageUrls);
var conflicts = ExtractConflicts(linkset);
var confidence = ComputeConfidence(providedConfidence, conflicts);
var confidence = ComputeConfidence(linkset, providedConfidence, conflicts);
return (normalized, confidence, conflicts);
}
@@ -171,28 +171,56 @@ internal static class AdvisoryLinksetNormalization
continue;
}
// Preserve existing notes but map into stable reason codes where possible.
var key = note.Key.Trim();
var reason = key switch
{
"severity" => "severity-mismatch",
"ranges" => "affected-range-divergence",
"references" => "reference-clash",
"aliases" => "alias-inconsistency",
_ => "metadata-gap"
};
conflicts.Add(new AdvisoryLinksetConflict(
note.Key.Trim(),
note.Value.Trim(),
null));
Field: key,
Reason: reason,
Values: new[] { $"{key}:{note.Value.Trim()}" }));
}
return conflicts;
}
private static double? ComputeConfidence(double? providedConfidence, IReadOnlyList<AdvisoryLinksetConflict> conflicts)
private static double? ComputeConfidence(RawLinkset linkset, double? providedConfidence, IReadOnlyList<AdvisoryLinksetConflict> conflicts)
{
if (providedConfidence.HasValue)
{
return CoerceConfidence(providedConfidence);
}
if (conflicts.Count > 0)
double aliasScore = linkset.Aliases.IsDefaultOrEmpty ? 0d : 1d;
double purlOverlapScore = linkset.PackageUrls.IsDefaultOrEmpty
? 0d
: (linkset.PackageUrls.Length > 1 ? 1d : 0.6d);
double cpeOverlapScore = linkset.Cpes.IsDefaultOrEmpty
? 0d
: (linkset.Cpes.Length > 1 ? 1d : 0.5d);
double severityAgreement = conflicts.Any(c => c.Reason == "severity-mismatch") ? 0.2d : 0.5d;
double referenceOverlap = linkset.References.IsDefaultOrEmpty ? 0d : 0.5d;
double freshnessScore = 0.5d; // until fetchedAt spread is available
var confidence = (0.40 * aliasScore) +
(0.25 * purlOverlapScore) +
(0.15 * cpeOverlapScore) +
(0.10 * severityAgreement) +
(0.05 * referenceOverlap) +
(0.05 * freshnessScore);
if (conflicts.Count > 0 && confidence > 0.7d)
{
// Basic heuristic until scoring pipeline is wired: any conflicts => lower confidence.
return 0.5;
confidence -= 0.1d; // penalize non-empty conflict sets
}
return 1.0;
return Math.Clamp(confidence, 0d, 1d);
}
}

11
src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/AdvisoryObservationEventPublisherOptions.cs Normal file
View File

@@ -0,0 +1,11 @@
namespace StellaOps.Concelier.Core.Observations;
public sealed class AdvisoryObservationEventPublisherOptions
{
public bool Enabled { get; set; } = false;
public string Transport { get; set; } = "mongo"; // mongo|nats
public string? NatsUrl { get; set; }
public string Subject { get; set; } = "concelier.advisory.observation.updated.v1";
public string DeadLetterSubject { get; set; } = "concelier.advisory.observation.updated.dead.v1";
public string Stream { get; set; } = "CONCELIER_OBS";
}

105
src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/AdvisoryObservationUpdatedEvent.cs Normal file
View File

@@ -0,0 +1,105 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Security.Cryptography;
using System.Text;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Core.Observations;
/// <summary>
/// Contract-matching payload for <c>advisory.observation.updated@1</c> events.
/// </summary>
public sealed record AdvisoryObservationUpdatedEvent(
Guid EventId,
string TenantId,
string ObservationId,
string AdvisoryId,
AdvisoryObservationSource Source,
AdvisoryObservationLinksetSummary LinksetSummary,
string DocumentSha,
string ObservationHash,
DateTimeOffset IngestedAt,
string ReplayCursor,
string? SupersedesId = null,
string? TraceId = null)
{
public static AdvisoryObservationUpdatedEvent FromObservation(
AdvisoryObservation observation,
string? supersedesId,
string? traceId,
string? replayCursor = null)
{
ArgumentNullException.ThrowIfNull(observation);
var summary = BuildSummary(observation.Linkset, observation.RawLinkset);
var observationHash = ComputeObservationHash(observation);
var tenantUrn = observation.Tenant.StartsWith("urn:tenant:", StringComparison.Ordinal)
? observation.Tenant
: $"urn:tenant:{observation.Tenant}";
return new AdvisoryObservationUpdatedEvent(
EventId: Guid.NewGuid(),
TenantId: tenantUrn,
ObservationId: observation.ObservationId,
AdvisoryId: observation.Upstream.UpstreamId,
Source: observation.Source,
LinksetSummary: summary,
DocumentSha: observation.Upstream.ContentHash,
ObservationHash: observationHash,
IngestedAt: observation.CreatedAt,
ReplayCursor: replayCursor ?? observation.CreatedAt.ToUniversalTime().Ticks.ToString(),
SupersedesId: supersedesId,
TraceId: traceId);
}
private static AdvisoryObservationLinksetSummary BuildSummary(
AdvisoryObservationLinkset linkset,
RawLinkset rawLinkset)
{
ArgumentNullException.ThrowIfNull(linkset);
ArgumentNullException.ThrowIfNull(rawLinkset);
static ImmutableArray<string> SortSet(IEnumerable<string> values)
=> values.Where(static v => !string.IsNullOrWhiteSpace(v))
.Select(static v => v.Trim())
.OrderBy(static v => v, StringComparer.Ordinal)
.ToImmutableArray();
var relationships = rawLinkset.Relationships.Select(static rel => new AdvisoryObservationRelationshipSummary(
rel.Type,
rel.Source,
rel.Target,
rel.Provenance)).ToImmutableArray();
return new AdvisoryObservationLinksetSummary(
Aliases: SortSet(linkset.Aliases),
Purls: SortSet(linkset.Purls),
Cpes: SortSet(linkset.Cpes),
Scopes: SortSet(rawLinkset.Scopes),
Relationships: relationships);
}
private static string ComputeObservationHash(AdvisoryObservation observation)
{
var json = CanonicalJsonSerializer.Serialize(observation);
var bytes = Encoding.UTF8.GetBytes(json);
var hashBytes = SHA256.HashData(bytes);
return Convert.ToHexString(hashBytes).ToLowerInvariant();
}
}
public sealed record AdvisoryObservationLinksetSummary(
ImmutableArray<string> Aliases,
ImmutableArray<string> Purls,
ImmutableArray<string> Cpes,
ImmutableArray<string> Scopes,
ImmutableArray<AdvisoryObservationRelationshipSummary> Relationships);
public sealed record AdvisoryObservationRelationshipSummary(
string Type,
string Source,
string Target,
string? Provenance);

12
src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/IAdvisoryObservationEventOutbox.cs Normal file
View File

@@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Observations;
public interface IAdvisoryObservationEventOutbox
{
Task<IReadOnlyCollection<AdvisoryObservationUpdatedEvent>> DequeueAsync(int take, CancellationToken cancellationToken);
Task MarkPublishedAsync(Guid eventId, DateTimeOffset publishedAt, CancellationToken cancellationToken);
}

9
src/Concelier/__Libraries/StellaOps.Concelier.Core/Observations/IAdvisoryObservationEventPublisher.cs Normal file
View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Concelier.Core.Observations;
public interface IAdvisoryObservationEventPublisher
{
Task PublishAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken);
}

56
src/Concelier/__Libraries/StellaOps.Concelier.Models/CanonicalJsonSerializer.cs
View File

@@ -1,9 +1,10 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
using System.Collections.Generic;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
using StellaOps.Concelier.Models.Observations;
namespace StellaOps.Concelier.Models;
@@ -17,11 +18,11 @@ public static class CanonicalJsonSerializer
private static readonly IReadOnlyDictionary<Type, string[]> PropertyOrderOverrides = new Dictionary<Type, string[]>
{
{
typeof(AdvisoryProvenance),
new[]
{
"source",
{
typeof(AdvisoryProvenance),
new[]
{
"source",
"kind",
"value",
"decisionReason",
@@ -69,15 +70,30 @@ public static class CanonicalJsonSerializer
{
typeof(AdvisoryWeakness),
new[]
{
"taxonomy",
"identifier",
"name",
"uri",
"provenance",
}
},
};
{
"taxonomy",
"identifier",
"name",
"uri",
"provenance",
}
},
{
typeof(AdvisoryObservation),
new[]
{
"observationId",
"tenant",
"source",
"upstream",
"content",
"linkset",
"rawLinkset",
"createdAt",
"attributes",
}
},
};
public static string Serialize<T>(T value)
=> JsonSerializer.Serialize(value, CompactOptions);

1
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/MIGRATIONS.md
View File

@@ -27,6 +27,7 @@ This module owns the persistent shape of Concelier's MongoDB database. Upgrades
| `20251028_advisory_supersedes_backfill` | Renames legacy `advisory` collection to a read-only backup view and backfills `supersedes` chains across `advisory_raw`. |
| `20251028_advisory_raw_validator` | Applies Aggregation-Only Contract JSON schema validator to the `advisory_raw` collection with configurable enforcement level. |
| `20251104_advisory_observations_raw_linkset` | Backfills `rawLinkset` on `advisory_observations` using stored `advisory_raw` documents so canonical and raw projections co-exist for downstream policy joins. |
| `20251120_advisory_observation_events` | Creates `advisory_observation_events` collection with tenant/hash indexes for observation event fan-out (advisory.observation.updated@1). Includes optional `publishedAt` marker for transport outbox. |
| `20251117_advisory_linksets_tenant_lower` | Lowercases `advisory_linksets.tenantId` to align writes with lookup filters. |
## Operator Runbook

31
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureAdvisoryObservationEventCollectionMigration.cs Normal file
View File

@@ -0,0 +1,31 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
public sealed class EnsureAdvisoryObservationEventCollectionMigration : IMongoMigration
{
public string Id => "20251120_advisory_observation_events";
public string Description => "Ensure advisory_observation_events collection and indexes exist for observation event fan-out.";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
var collection = database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryObservationEvents);
var indexes = new List<CreateIndexModel<BsonDocument>>
{
new(
Builders<BsonDocument>.IndexKeys.Ascending("tenantId").Descending("ingestedAt"),
new CreateIndexOptions { Name = "advisory_observation_events_tenant_ingested_desc" }),
new(
Builders<BsonDocument>.IndexKeys.Ascending("observationHash"),
new CreateIndexOptions { Name = "advisory_observation_events_hash_unique", Unique = true }),
};
await collection.Indexes.CreateManyAsync(indexes, cancellationToken).ConfigureAwait(false);
}
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/MongoStorageDefaults.cs
View File

@@ -29,5 +29,6 @@ public static class MongoStorageDefaults
public const string AdvisoryConflicts = "advisory_conflicts";
public const string AdvisoryObservations = "advisory_observations";
public const string AdvisoryLinksets = "advisory_linksets";
public const string AdvisoryObservationEvents = "advisory_observation_events";
}
}

97
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationEventDocument.cs Normal file
View File

@@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
public sealed class AdvisoryObservationEventDocument
{
[BsonId]
public Guid Id { get; set; }
[BsonElement("tenantId")]
public string TenantId { get; set; } = string.Empty;
[BsonElement("observationId")]
public string ObservationId { get; set; } = string.Empty;
[BsonElement("advisoryId")]
public string AdvisoryId { get; set; } = string.Empty;
[BsonElement("source")]
public AdvisoryObservationSourceDocument Source { get; set; } = new();
[BsonElement("linksetSummary")]
public AdvisoryObservationLinksetSummaryDocument LinksetSummary { get; set; } = new();
[BsonElement("documentSha")]
public string DocumentSha { get; set; } = string.Empty;
[BsonElement("observationHash")]
public string ObservationHash { get; set; } = string.Empty;
[BsonElement("ingestedAt")]
public DateTime IngestedAt { get; set; }
= DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
[BsonElement("replayCursor")]
public string ReplayCursor { get; set; } = string.Empty;
[BsonElement("supersedesId")]
public string? SupersedesId { get; set; }
[BsonElement("traceId")]
public string? TraceId { get; set; }
[BsonElement("publishedAt")]
public DateTime? PublishedAt { get; set; }
}
public sealed class AdvisoryObservationSourceDocument
{
[BsonElement("vendor")]
public string Vendor { get; set; } = string.Empty;
[BsonElement("stream")]
public string Stream { get; set; } = string.Empty;
[BsonElement("api")]
public string Api { get; set; } = string.Empty;
[BsonElement("collectorVersion")]
public string? CollectorVersion { get; set; }
}
public sealed class AdvisoryObservationLinksetSummaryDocument
{
[BsonElement("aliases")]
public List<string> Aliases { get; set; } = new();
[BsonElement("purls")]
public List<string> Purls { get; set; } = new();
[BsonElement("cpes")]
public List<string> Cpes { get; set; } = new();
[BsonElement("scopes")]
public List<string> Scopes { get; set; } = new();
[BsonElement("relationships")]
public List<AdvisoryObservationRelationshipDocument> Relationships { get; set; } = new();
}
public sealed class AdvisoryObservationRelationshipDocument
{
[BsonElement("type")]
public string Type { get; set; } = string.Empty;
[BsonElement("source")]
public string Source { get; set; } = string.Empty;
[BsonElement("target")]
public string Target { get; set; } = string.Empty;
[BsonElement("provenance")]
public string? Provenance { get; set; }
}

28
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationSink.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Observations;
@@ -9,14 +10,37 @@ namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class AdvisoryObservationSink : IAdvisoryObservationSink
{
private readonly IAdvisoryObservationStore _store;
private readonly IAdvisoryObservationEventPublisher _publisher;
private readonly TimeProvider _timeProvider;
public AdvisoryObservationSink(IAdvisoryObservationStore store)
public AdvisoryObservationSink(
IAdvisoryObservationStore store,
IAdvisoryObservationEventPublisher publisher,
TimeProvider? timeProvider = null)
{
_store = store ?? throw new ArgumentNullException(nameof(store));
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken)
{
return _store.UpsertAsync(observation, cancellationToken);
ArgumentNullException.ThrowIfNull(observation);
return UpsertAndPublishAsync(observation, cancellationToken);
}
private async Task UpsertAndPublishAsync(AdvisoryObservation observation, CancellationToken cancellationToken)
{
await _store.UpsertAsync(observation, cancellationToken).ConfigureAwait(false);
var evt = AdvisoryObservationUpdatedEvent.FromObservation(
observation,
supersedesId: observation.Attributes.GetValueOrDefault("supersedesId")
?? observation.Attributes.GetValueOrDefault("supersedes"),
traceId: observation.Attributes.GetValueOrDefault("traceId"),
replayCursor: _timeProvider.GetUtcNow().Ticks.ToString());
await _publisher.PublishAsync(evt, cancellationToken).ConfigureAwait(false);
}
}

66
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationTransportWorker.cs Normal file
View File

@@ -0,0 +1,66 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class AdvisoryObservationTransportWorker : BackgroundService
{
private readonly IAdvisoryObservationEventOutbox _outbox;
private readonly IAdvisoryObservationEventPublisher _publisher;
private readonly ILogger<AdvisoryObservationTransportWorker> _logger;
private readonly AdvisoryObservationEventPublisherOptions _options;
public AdvisoryObservationTransportWorker(
IAdvisoryObservationEventOutbox outbox,
IAdvisoryObservationEventPublisher publisher,
IOptions<AdvisoryObservationEventPublisherOptions> options,
ILogger<AdvisoryObservationTransportWorker> logger)
{
_outbox = outbox ?? throw new ArgumentNullException(nameof(outbox));
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = options.Value;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (!_options.Enabled)
{
_logger.LogInformation("Observation transport worker disabled.");
return;
}
while (!stoppingToken.IsCancellationRequested)
{
try
{
var batch = await _outbox.DequeueAsync(25, stoppingToken).ConfigureAwait(false);
if (batch.Count == 0)
{
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var evt in batch)
{
await _publisher.PublishAsync(evt, stoppingToken).ConfigureAwait(false);
await _outbox.MarkPublishedAsync(evt.EventId, DateTimeOffset.UtcNow, stoppingToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Observation transport worker error; retrying");
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
}
}

74
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Observations/MongoAdvisoryObservationEventOutbox.cs Normal file
View File

@@ -0,0 +1,74 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class MongoAdvisoryObservationEventOutbox : IAdvisoryObservationEventOutbox
{
private readonly IMongoCollection<AdvisoryObservationEventDocument> _collection;
public MongoAdvisoryObservationEventOutbox(IMongoCollection<AdvisoryObservationEventDocument> collection)
{
_collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
public async Task<IReadOnlyCollection<AdvisoryObservationUpdatedEvent>> DequeueAsync(int take, CancellationToken cancellationToken)
{
if (take <= 0)
{
return Array.Empty<AdvisoryObservationUpdatedEvent>();
}
var filter = Builders<AdvisoryObservationEventDocument>.Filter.Eq(doc => doc.PublishedAt, null);
var documents = await _collection
.Find(filter)
.SortByDescending(doc => doc.IngestedAt)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents.Select(ToDomain).ToArray();
}
public Task MarkPublishedAsync(Guid eventId, DateTimeOffset publishedAt, CancellationToken cancellationToken)
{
var update = Builders<AdvisoryObservationEventDocument>.Update.Set(doc => doc.PublishedAt, publishedAt.UtcDateTime);
return _collection.UpdateOneAsync(
Builders<AdvisoryObservationEventDocument>.Filter.Eq(doc => doc.Id, eventId),
update,
cancellationToken: cancellationToken);
}
private static AdvisoryObservationUpdatedEvent ToDomain(AdvisoryObservationEventDocument doc)
{
return new AdvisoryObservationUpdatedEvent(
doc.Id,
doc.TenantId,
doc.ObservationId,
doc.AdvisoryId,
new Models.Observations.AdvisoryObservationSource(
doc.Source.Vendor,
doc.Source.Stream,
doc.Source.Api,
doc.Source.CollectorVersion),
new AdvisoryObservationLinksetSummary(
doc.LinksetSummary.Aliases.ToImmutableArray(),
doc.LinksetSummary.Purls.ToImmutableArray(),
doc.LinksetSummary.Cpes.ToImmutableArray(),
doc.LinksetSummary.Scopes.ToImmutableArray(),
doc.LinksetSummary.Relationships
.Select(rel => new AdvisoryObservationRelationshipSummary(rel.Type, rel.Source, rel.Target, rel.Provenance))
.ToImmutableArray()),
doc.DocumentSha,
doc.ObservationHash,
doc.IngestedAt,
doc.ReplayCursor,
doc.SupersedesId,
doc.TraceId);
}
}

62
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Observations/MongoAdvisoryObservationEventPublisher.cs Normal file
View File

@@ -0,0 +1,62 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class MongoAdvisoryObservationEventPublisher : IAdvisoryObservationEventPublisher
{
private readonly IMongoCollection<AdvisoryObservationEventDocument> _collection;
public MongoAdvisoryObservationEventPublisher(IMongoCollection<AdvisoryObservationEventDocument> collection)
{
_collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
public Task PublishAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(@event);
var document = new AdvisoryObservationEventDocument
{
Id = @event.EventId,
TenantId = @event.TenantId,
ObservationId = @event.ObservationId,
AdvisoryId = @event.AdvisoryId,
DocumentSha = @event.DocumentSha,
ObservationHash = @event.ObservationHash,
IngestedAt = @event.IngestedAt.UtcDateTime,
ReplayCursor = @event.ReplayCursor,
SupersedesId = @event.SupersedesId,
TraceId = @event.TraceId,
Source = new AdvisoryObservationSourceDocument
{
Vendor = @event.Source.Vendor,
Stream = @event.Source.Stream,
Api = @event.Source.Api,
CollectorVersion = @event.Source.CollectorVersion
},
LinksetSummary = new AdvisoryObservationLinksetSummaryDocument
{
Aliases = @event.LinksetSummary.Aliases.ToList(),
Purls = @event.LinksetSummary.Purls.ToList(),
Cpes = @event.LinksetSummary.Cpes.ToList(),
Scopes = @event.LinksetSummary.Scopes.ToList(),
Relationships = @event.LinksetSummary.Relationships
.Select(static rel => new AdvisoryObservationRelationshipDocument
{
Type = rel.Type,
Source = rel.Source,
Target = rel.Target,
Provenance = rel.Provenance
})
.ToList()
}
};
return _collection.InsertOneAsync(document, cancellationToken: cancellationToken);
}
}

66
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/Observations/NatsAdvisoryObservationEventPublisher.cs Normal file
View File

@@ -0,0 +1,66 @@
using System;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using NATS.Client.Core;
using NATS.Client.JetStream;
using StellaOps.Concelier.Core.Observations;
namespace StellaOps.Concelier.Storage.Mongo.Observations;
internal sealed class NatsAdvisoryObservationEventPublisher : IAdvisoryObservationEventPublisher
{
private readonly ILogger<NatsAdvisoryObservationEventPublisher> _logger;
private readonly AdvisoryObservationEventPublisherOptions _options;
public NatsAdvisoryObservationEventPublisher(
IOptions<AdvisoryObservationEventPublisherOptions> options,
ILogger<NatsAdvisoryObservationEventPublisher> logger)
{
ArgumentNullException.ThrowIfNull(options);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = options.Value;
}
public async Task PublishAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken)
{
if (!_options.Enabled)
{
return;
}
var subject = _options.Subject;
var payload = JsonSerializer.SerializeToUtf8Bytes(@event);
var opts = new NatsOpts { Url = _options.NatsUrl ?? "nats://127.0.0.1:4222" };
await using var connection = new NatsConnection(opts);
var js = new NatsJSContext(connection);
await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false);
await js.PublishAsync(subject, payload, cancellationToken: cancellationToken).ConfigureAwait(false);
_logger.LogDebug("Published advisory.observation.updated@1 to NATS subject {Subject} for observation {ObservationId}", subject, @event.ObservationId);
}
private async Task EnsureStreamAsync(INatsJSContext js, CancellationToken cancellationToken)
{
var stream = _options.Stream;
try
{
await js.GetStreamAsync(stream, cancellationToken).ConfigureAwait(false);
}
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
{
var cfg = new NatsJSStreamConfig
{
Name = stream,
Subjects = new[] { _options.Subject },
Description = "Concelier advisory observation events",
MaxMsgSize = 512 * 1024,
};
await js.CreateStreamAsync(cfg, cancellationToken).ConfigureAwait(false);
}
}
}

22
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs
View File

@@ -79,6 +79,19 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IAdvisoryObservationLookup, AdvisoryObservationLookup>();
services.AddSingleton<IAdvisoryEventRepository, MongoAdvisoryEventRepository>();
services.AddSingleton<IAdvisoryEventLog, AdvisoryEventLog>();
services.AddSingleton<MongoAdvisoryObservationEventPublisher>();
services.AddSingleton<NatsAdvisoryObservationEventPublisher>();
services.AddSingleton<IAdvisoryObservationEventPublisher>(sp =>
{
var options = sp.GetRequiredService<IOptions<AdvisoryObservationEventPublisherOptions>>().Value;
if (string.Equals(options.Transport, "nats", StringComparison.OrdinalIgnoreCase))
{
return sp.GetRequiredService<NatsAdvisoryObservationEventPublisher>();
}
return sp.GetRequiredService<MongoAdvisoryObservationEventPublisher>();
});
services.AddSingleton<IAdvisoryObservationEventOutbox, MongoAdvisoryObservationEventOutbox>();
services.AddSingleton<IAdvisoryRawRepository, MongoAdvisoryRawRepository>();
services.AddSingleton<StellaOps.Concelier.Storage.Mongo.Linksets.IMongoAdvisoryLinksetStore, StellaOps.Concelier.Storage.Mongo.Linksets.ConcelierMongoLinksetStore>();
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetStore>(sp =>
@@ -108,6 +121,12 @@ public static class ServiceCollectionExtensions
return database.GetCollection<AdvisoryObservationDocument>(MongoStorageDefaults.Collections.AdvisoryObservations);
});
services.AddSingleton<IMongoCollection<AdvisoryObservationEventDocument>>(static sp =>
{
var database = sp.GetRequiredService<IMongoDatabase>();
return database.GetCollection<AdvisoryObservationEventDocument>(MongoStorageDefaults.Collections.AdvisoryObservationEvents);
});
services.AddSingleton<IMongoCollection<AdvisoryLinksetDocument>>(static sp =>
{
var database = sp.GetRequiredService<IMongoDatabase>();
@@ -126,8 +145,11 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IMongoMigration, EnsureAdvisoryObservationsRawLinksetMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryLinksetsTenantLowerMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryEventCollectionsMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryObservationEventCollectionMigration>();
services.AddSingleton<IMongoMigration, SemVerStyleBackfillMigration>();
services.AddSingleton<IHostedService, AdvisoryObservationTransportWorker>();
return services;
}
}

11
src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj
View File

@@ -6,11 +6,12 @@
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="NATS.Client.Core" Version="2.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />

31
src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/AdvisoryLinksetNormalizationConfidenceTests.cs Normal file
View File

@@ -0,0 +1,31 @@
using System.Collections.Immutable;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.RawModels;
using Xunit;
namespace StellaOps.Concelier.Core.Tests.Linksets;
public sealed class AdvisoryLinksetNormalizationConfidenceTests
{
[Fact]
public void FromRawLinksetWithConfidence_ComputesWeightedScoreAndReasons()
{
var linkset = new RawLinkset
{
Aliases = ImmutableArray.Create("CVE-2024-11111", "GHSA-aaaa-bbbb"),
PackageUrls = ImmutableArray.Create("pkg:npm/foo@1.0.0", "pkg:npm/foo@1.1.0"),
Cpes = ImmutableArray.Create("cpe:/a:foo:foo:1.0.0", "cpe:/a:foo:foo:1.1.0"),
Notes = ImmutableDictionary.CreateRange(new[] { new KeyValuePair<string, string>("severity", "mismatch") })
};
var (normalized, confidence, conflicts) = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(linkset);
Assert.NotNull(normalized);
Assert.NotNull(confidence);
Assert.True(confidence!.Value is > 0.7 and < 0.8); // weighted score with conflict penalty
var conflict = Assert.Single(conflicts);
Assert.Equal("severity-mismatch", conflict.Reason);
Assert.Contains("severity:mismatch", conflict.Values!);
}
}

2
src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Observations/AdvisoryObservationAggregationTests.cs
View File

@@ -53,7 +53,7 @@ public sealed class AdvisoryObservationAggregationTests
var confidence = result.confidence;
var conflicts = result.conflicts;
Assert.Equal(0.5, confidence);
Assert.True(confidence is >= 0.1 and <= 0.6);
Assert.Single(conflicts);
Assert.Null(normalized); // no purls supplied
}

68
src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Observations/AdvisoryObservationEventFactoryTests.cs Normal file
View File

@@ -0,0 +1,68 @@
using System;
using System.Collections.Immutable;
using System.Text.Json.Nodes;
using StellaOps.Concelier.Core.Observations;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
using Xunit;
namespace StellaOps.Concelier.Core.Tests.Observations;
public sealed class AdvisoryObservationEventFactoryTests
{
[Fact]
public void FromObservation_MapsFieldsAndHashesDeterministically()
{
var observation = CreateObservation();
var evt = AdvisoryObservationUpdatedEvent.FromObservation(
observation,
supersedesId: "655fabcdedc0ffee0000abcd",
traceId: "trace-123");
Assert.Equal("urn:tenant:tenant-1", evt.TenantId);
Assert.Equal("adv-1", evt.AdvisoryId);
Assert.Equal("655fabcdedc0ffee0000abcd", evt.SupersedesId);
Assert.NotNull(evt.ObservationHash);
Assert.Equal(observation.Upstream.ContentHash, evt.DocumentSha);
Assert.Contains("pkg:npm/foo", evt.LinksetSummary.Purls);
}
private static AdvisoryObservation CreateObservation()
{
var source = new AdvisoryObservationSource("ghsa", "advisories", "https://api");
var upstream = new AdvisoryObservationUpstream(
"adv-1",
"v1",
DateTimeOffset.Parse("2025-11-20T12:00:00Z"),
DateTimeOffset.Parse("2025-11-20T12:00:00Z"),
"2f8f568cc1ed3474f0a4564ddb8c64f4b4d176fbe0a2a98a02b88e822a4f5b6d",
new AdvisoryObservationSignature(false, null, null, null));
var content = new AdvisoryObservationContent("json", null, JsonNode.Parse("{}")!);
var linkset = new AdvisoryObservationLinkset(
aliases: new[] { "CVE-2024-1234", "GHSA-xxxx" },
purls: new[] { "pkg:npm/foo@1.0.0" },
cpes: new[] { "cpe:/a:foo:foo:1.0.0" },
references: new[] { new AdvisoryObservationReference("ref", "https://example.com") });
var rawLinkset = new RawLinkset
{
Aliases = ImmutableArray.Create("CVE-2024-1234", "GHSA-xxxx"),
PackageUrls = ImmutableArray.Create("pkg:npm/foo@1.0.0"),
Cpes = ImmutableArray.Create("cpe:/a:foo:foo:1.0.0"),
Scopes = ImmutableArray.Create("runtime"),
Relationships = ImmutableArray.Create(new RawRelationship("contains", "pkg:npm/foo@1.0.0", "file://dist/foo.js")),
};
return new AdvisoryObservation(
"655fabcdf3c5d6ad3b5a0aaa",
"tenant-1",
source,
upstream,
content,
linkset,
rawLinkset,
DateTimeOffset.Parse("2025-11-20T12:01:00Z"));
}
}