feat: Enhance MongoDB storage with event publishing and outbox support
- Added `MongoAdvisoryObservationEventPublisher` and `NatsAdvisoryObservationEventPublisher` for event publishing. - Registered `IAdvisoryObservationEventPublisher` to choose between NATS and MongoDB based on configuration. - Introduced `MongoAdvisoryObservationEventOutbox` for outbox pattern implementation. - Updated service collection to include new event publishers and outbox. - Added a new hosted service `AdvisoryObservationTransportWorker` for processing events. feat: Update project dependencies - Added `NATS.Client.Core` package to the project for NATS integration. test: Add unit tests for AdvisoryLinkset normalization - Created `AdvisoryLinksetNormalizationConfidenceTests` to validate confidence score calculations. fix: Adjust confidence assertion in `AdvisoryObservationAggregationTests` - Updated confidence assertion to allow a range instead of a fixed value. test: Implement tests for AdvisoryObservationEventFactory - Added `AdvisoryObservationEventFactoryTests` to ensure correct mapping and hashing of observation events. chore: Configure test project for Findings Ledger - Created `Directory.Build.props` for test project configuration. - Added `StellaOps.Findings.Ledger.Exports.Unit.csproj` for unit tests related to findings ledger exports. feat: Implement export contracts for findings ledger - Defined export request and response contracts in `ExportContracts.cs`. - Created various export item records for findings, VEX, advisories, and SBOMs. feat: Add export functionality to Findings Ledger Web Service - Implemented endpoints for exporting findings, VEX, advisories, and SBOMs. - Integrated `ExportQueryService` for handling export logic and pagination. test: Add tests for Node language analyzer phase 22 - Implemented `NodePhase22SampleLoaderTests` to validate loading of NDJSON fixtures. - Created sample NDJSON file for testing. chore: Set up isolated test environment for Node tests - Added `node-isolated.runsettings` for isolated test execution. - Created `node-tests-isolated.sh` script for running tests in isolation.
This commit is contained in:
master
@@ -106,6 +106,15 @@ builder.Services.AddMongoStorage(storageOptions =>
|
||||
storageOptions.DatabaseName = concelierOptions.Storage.Database;
|
||||
storageOptions.CommandTimeout = TimeSpan.FromSeconds(concelierOptions.Storage.CommandTimeoutSeconds);
|
||||
});
|
||||
builder.Services.AddOptions<AdvisoryObservationEventPublisherOptions>()
|
||||
.Bind(builder.Configuration.GetSection("advisoryObservationEvents"))
|
||||
.PostConfigure(options =>
|
||||
{
|
||||
options.Subject ??= "concelier.advisory.observation.updated.v1";
|
||||
options.Stream ??= "CONCELIER_OBS";
|
||||
options.Transport = string.IsNullOrWhiteSpace(options.Transport) ? "mongo" : options.Transport;
|
||||
})
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddConcelierAocGuards();
|
||||
builder.Services.AddConcelierLinksetMappers();
|
||||
builder.Services.AddAdvisoryRawServices();
|
||||
|
||||
@@ -33,7 +33,7 @@ internal static class AdvisoryLinksetNormalization
|
||||
|
||||
var normalized = Build(linkset.PackageUrls);
|
||||
var conflicts = ExtractConflicts(linkset);
|
||||
var confidence = ComputeConfidence(providedConfidence, conflicts);
|
||||
var confidence = ComputeConfidence(linkset, providedConfidence, conflicts);
|
||||
|
||||
return (normalized, confidence, conflicts);
|
||||
}
|
||||
@@ -171,28 +171,56 @@ internal static class AdvisoryLinksetNormalization
|
||||
continue;
|
||||
}
|
||||
|
||||
// Preserve existing notes but map into stable reason codes where possible.
|
||||
var key = note.Key.Trim();
|
||||
var reason = key switch
|
||||
{
|
||||
"severity" => "severity-mismatch",
|
||||
"ranges" => "affected-range-divergence",
|
||||
"references" => "reference-clash",
|
||||
"aliases" => "alias-inconsistency",
|
||||
_ => "metadata-gap"
|
||||
};
|
||||
|
||||
conflicts.Add(new AdvisoryLinksetConflict(
|
||||
note.Key.Trim(),
|
||||
note.Value.Trim(),
|
||||
null));
|
||||
Field: key,
|
||||
Reason: reason,
|
||||
Values: new[] { $"{key}:{note.Value.Trim()}" }));
|
||||
}
|
||||
|
||||
return conflicts;
|
||||
}
|
||||
|
||||
private static double? ComputeConfidence(double? providedConfidence, IReadOnlyList<AdvisoryLinksetConflict> conflicts)
|
||||
private static double? ComputeConfidence(RawLinkset linkset, double? providedConfidence, IReadOnlyList<AdvisoryLinksetConflict> conflicts)
|
||||
{
|
||||
if (providedConfidence.HasValue)
|
||||
{
|
||||
return CoerceConfidence(providedConfidence);
|
||||
}
|
||||
|
||||
if (conflicts.Count > 0)
|
||||
double aliasScore = linkset.Aliases.IsDefaultOrEmpty ? 0d : 1d;
|
||||
double purlOverlapScore = linkset.PackageUrls.IsDefaultOrEmpty
|
||||
? 0d
|
||||
: (linkset.PackageUrls.Length > 1 ? 1d : 0.6d);
|
||||
double cpeOverlapScore = linkset.Cpes.IsDefaultOrEmpty
|
||||
? 0d
|
||||
: (linkset.Cpes.Length > 1 ? 1d : 0.5d);
|
||||
double severityAgreement = conflicts.Any(c => c.Reason == "severity-mismatch") ? 0.2d : 0.5d;
|
||||
double referenceOverlap = linkset.References.IsDefaultOrEmpty ? 0d : 0.5d;
|
||||
double freshnessScore = 0.5d; // until fetchedAt spread is available
|
||||
|
||||
var confidence = (0.40 * aliasScore) +
|
||||
(0.25 * purlOverlapScore) +
|
||||
(0.15 * cpeOverlapScore) +
|
||||
(0.10 * severityAgreement) +
|
||||
(0.05 * referenceOverlap) +
|
||||
(0.05 * freshnessScore);
|
||||
|
||||
if (conflicts.Count > 0 && confidence > 0.7d)
|
||||
{
|
||||
// Basic heuristic until scoring pipeline is wired: any conflicts => lower confidence.
|
||||
return 0.5;
|
||||
confidence -= 0.1d; // penalize non-empty conflict sets
|
||||
}
|
||||
|
||||
return 1.0;
|
||||
return Math.Clamp(confidence, 0d, 1d);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace StellaOps.Concelier.Core.Observations;
|
||||
|
||||
public sealed class AdvisoryObservationEventPublisherOptions
|
||||
{
|
||||
public bool Enabled { get; set; } = false;
|
||||
public string Transport { get; set; } = "mongo"; // mongo|nats
|
||||
public string? NatsUrl { get; set; }
|
||||
public string Subject { get; set; } = "concelier.advisory.observation.updated.v1";
|
||||
public string DeadLetterSubject { get; set; } = "concelier.advisory.observation.updated.dead.v1";
|
||||
public string Stream { get; set; } = "CONCELIER_OBS";
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Models.Observations;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Contract-matching payload for <c>advisory.observation.updated@1</c> events.
|
||||
/// </summary>
|
||||
public sealed record AdvisoryObservationUpdatedEvent(
|
||||
Guid EventId,
|
||||
string TenantId,
|
||||
string ObservationId,
|
||||
string AdvisoryId,
|
||||
AdvisoryObservationSource Source,
|
||||
AdvisoryObservationLinksetSummary LinksetSummary,
|
||||
string DocumentSha,
|
||||
string ObservationHash,
|
||||
DateTimeOffset IngestedAt,
|
||||
string ReplayCursor,
|
||||
string? SupersedesId = null,
|
||||
string? TraceId = null)
|
||||
{
|
||||
public static AdvisoryObservationUpdatedEvent FromObservation(
|
||||
AdvisoryObservation observation,
|
||||
string? supersedesId,
|
||||
string? traceId,
|
||||
string? replayCursor = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(observation);
|
||||
|
||||
var summary = BuildSummary(observation.Linkset, observation.RawLinkset);
|
||||
var observationHash = ComputeObservationHash(observation);
|
||||
var tenantUrn = observation.Tenant.StartsWith("urn:tenant:", StringComparison.Ordinal)
|
||||
? observation.Tenant
|
||||
: $"urn:tenant:{observation.Tenant}";
|
||||
|
||||
return new AdvisoryObservationUpdatedEvent(
|
||||
EventId: Guid.NewGuid(),
|
||||
TenantId: tenantUrn,
|
||||
ObservationId: observation.ObservationId,
|
||||
AdvisoryId: observation.Upstream.UpstreamId,
|
||||
Source: observation.Source,
|
||||
LinksetSummary: summary,
|
||||
DocumentSha: observation.Upstream.ContentHash,
|
||||
ObservationHash: observationHash,
|
||||
IngestedAt: observation.CreatedAt,
|
||||
ReplayCursor: replayCursor ?? observation.CreatedAt.ToUniversalTime().Ticks.ToString(),
|
||||
SupersedesId: supersedesId,
|
||||
TraceId: traceId);
|
||||
}
|
||||
|
||||
private static AdvisoryObservationLinksetSummary BuildSummary(
|
||||
AdvisoryObservationLinkset linkset,
|
||||
RawLinkset rawLinkset)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(linkset);
|
||||
ArgumentNullException.ThrowIfNull(rawLinkset);
|
||||
|
||||
static ImmutableArray<string> SortSet(IEnumerable<string> values)
|
||||
=> values.Where(static v => !string.IsNullOrWhiteSpace(v))
|
||||
.Select(static v => v.Trim())
|
||||
.OrderBy(static v => v, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var relationships = rawLinkset.Relationships.Select(static rel => new AdvisoryObservationRelationshipSummary(
|
||||
rel.Type,
|
||||
rel.Source,
|
||||
rel.Target,
|
||||
rel.Provenance)).ToImmutableArray();
|
||||
|
||||
return new AdvisoryObservationLinksetSummary(
|
||||
Aliases: SortSet(linkset.Aliases),
|
||||
Purls: SortSet(linkset.Purls),
|
||||
Cpes: SortSet(linkset.Cpes),
|
||||
Scopes: SortSet(rawLinkset.Scopes),
|
||||
Relationships: relationships);
|
||||
}
|
||||
|
||||
private static string ComputeObservationHash(AdvisoryObservation observation)
|
||||
{
|
||||
var json = CanonicalJsonSerializer.Serialize(observation);
|
||||
var bytes = Encoding.UTF8.GetBytes(json);
|
||||
var hashBytes = SHA256.HashData(bytes);
|
||||
return Convert.ToHexString(hashBytes).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record AdvisoryObservationLinksetSummary(
|
||||
ImmutableArray<string> Aliases,
|
||||
ImmutableArray<string> Purls,
|
||||
ImmutableArray<string> Cpes,
|
||||
ImmutableArray<string> Scopes,
|
||||
ImmutableArray<AdvisoryObservationRelationshipSummary> Relationships);
|
||||
|
||||
public sealed record AdvisoryObservationRelationshipSummary(
|
||||
string Type,
|
||||
string Source,
|
||||
string Target,
|
||||
string? Provenance);
|
||||
@@ -0,0 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Observations;
|
||||
|
||||
public interface IAdvisoryObservationEventOutbox
|
||||
{
|
||||
Task<IReadOnlyCollection<AdvisoryObservationUpdatedEvent>> DequeueAsync(int take, CancellationToken cancellationToken);
|
||||
Task MarkPublishedAsync(Guid eventId, DateTimeOffset publishedAt, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Observations;
|
||||
|
||||
public interface IAdvisoryObservationEventPublisher
|
||||
{
|
||||
Task PublishAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.Json.Serialization.Metadata;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.Json.Serialization.Metadata;
|
||||
using StellaOps.Concelier.Models.Observations;
|
||||
|
||||
namespace StellaOps.Concelier.Models;
|
||||
|
||||
@@ -17,11 +18,11 @@ public static class CanonicalJsonSerializer
|
||||
|
||||
private static readonly IReadOnlyDictionary<Type, string[]> PropertyOrderOverrides = new Dictionary<Type, string[]>
|
||||
{
|
||||
{
|
||||
typeof(AdvisoryProvenance),
|
||||
new[]
|
||||
{
|
||||
"source",
|
||||
{
|
||||
typeof(AdvisoryProvenance),
|
||||
new[]
|
||||
{
|
||||
"source",
|
||||
"kind",
|
||||
"value",
|
||||
"decisionReason",
|
||||
@@ -69,15 +70,30 @@ public static class CanonicalJsonSerializer
|
||||
{
|
||||
typeof(AdvisoryWeakness),
|
||||
new[]
|
||||
{
|
||||
"taxonomy",
|
||||
"identifier",
|
||||
"name",
|
||||
"uri",
|
||||
"provenance",
|
||||
}
|
||||
},
|
||||
};
|
||||
{
|
||||
"taxonomy",
|
||||
"identifier",
|
||||
"name",
|
||||
"uri",
|
||||
"provenance",
|
||||
}
|
||||
},
|
||||
{
|
||||
typeof(AdvisoryObservation),
|
||||
new[]
|
||||
{
|
||||
"observationId",
|
||||
"tenant",
|
||||
"source",
|
||||
"upstream",
|
||||
"content",
|
||||
"linkset",
|
||||
"rawLinkset",
|
||||
"createdAt",
|
||||
"attributes",
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
public static string Serialize<T>(T value)
|
||||
=> JsonSerializer.Serialize(value, CompactOptions);
|
||||
|
||||
@@ -27,6 +27,7 @@ This module owns the persistent shape of Concelier's MongoDB database. Upgrades
|
||||
| `20251028_advisory_supersedes_backfill` | Renames legacy `advisory` collection to a read-only backup view and backfills `supersedes` chains across `advisory_raw`. |
|
||||
| `20251028_advisory_raw_validator` | Applies Aggregation-Only Contract JSON schema validator to the `advisory_raw` collection with configurable enforcement level. |
|
||||
| `20251104_advisory_observations_raw_linkset` | Backfills `rawLinkset` on `advisory_observations` using stored `advisory_raw` documents so canonical and raw projections co-exist for downstream policy joins. |
|
||||
| `20251120_advisory_observation_events` | Creates `advisory_observation_events` collection with tenant/hash indexes for observation event fan-out (advisory.observation.updated@1). Includes optional `publishedAt` marker for transport outbox. |
|
||||
| `20251117_advisory_linksets_tenant_lower` | Lowercases `advisory_linksets.tenantId` to align writes with lookup filters. |
|
||||
|
||||
## Operator Runbook
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
|
||||
|
||||
public sealed class EnsureAdvisoryObservationEventCollectionMigration : IMongoMigration
|
||||
{
|
||||
public string Id => "20251120_advisory_observation_events";
|
||||
|
||||
public string Description => "Ensure advisory_observation_events collection and indexes exist for observation event fan-out.";
|
||||
|
||||
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryObservationEvents);
|
||||
|
||||
var indexes = new List<CreateIndexModel<BsonDocument>>
|
||||
{
|
||||
new(
|
||||
Builders<BsonDocument>.IndexKeys.Ascending("tenantId").Descending("ingestedAt"),
|
||||
new CreateIndexOptions { Name = "advisory_observation_events_tenant_ingested_desc" }),
|
||||
new(
|
||||
Builders<BsonDocument>.IndexKeys.Ascending("observationHash"),
|
||||
new CreateIndexOptions { Name = "advisory_observation_events_hash_unique", Unique = true }),
|
||||
};
|
||||
|
||||
await collection.Indexes.CreateManyAsync(indexes, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -29,5 +29,6 @@ public static class MongoStorageDefaults
|
||||
public const string AdvisoryConflicts = "advisory_conflicts";
|
||||
public const string AdvisoryObservations = "advisory_observations";
|
||||
public const string AdvisoryLinksets = "advisory_linksets";
|
||||
public const string AdvisoryObservationEvents = "advisory_observation_events";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Mongo.Observations;
|
||||
|
||||
public sealed class AdvisoryObservationEventDocument
|
||||
{
|
||||
[BsonId]
|
||||
public Guid Id { get; set; }
|
||||
|
||||
[BsonElement("tenantId")]
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("observationId")]
|
||||
public string ObservationId { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("advisoryId")]
|
||||
public string AdvisoryId { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("source")]
|
||||
public AdvisoryObservationSourceDocument Source { get; set; } = new();
|
||||
|
||||
[BsonElement("linksetSummary")]
|
||||
public AdvisoryObservationLinksetSummaryDocument LinksetSummary { get; set; } = new();
|
||||
|
||||
[BsonElement("documentSha")]
|
||||
public string DocumentSha { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("observationHash")]
|
||||
public string ObservationHash { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("ingestedAt")]
|
||||
public DateTime IngestedAt { get; set; }
|
||||
= DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
|
||||
|
||||
[BsonElement("replayCursor")]
|
||||
public string ReplayCursor { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("supersedesId")]
|
||||
public string? SupersedesId { get; set; }
|
||||
|
||||
[BsonElement("traceId")]
|
||||
public string? TraceId { get; set; }
|
||||
|
||||
[BsonElement("publishedAt")]
|
||||
public DateTime? PublishedAt { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AdvisoryObservationSourceDocument
|
||||
{
|
||||
[BsonElement("vendor")]
|
||||
public string Vendor { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("stream")]
|
||||
public string Stream { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("api")]
|
||||
public string Api { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("collectorVersion")]
|
||||
public string? CollectorVersion { get; set; }
|
||||
}
|
||||
|
||||
public sealed class AdvisoryObservationLinksetSummaryDocument
|
||||
{
|
||||
[BsonElement("aliases")]
|
||||
public List<string> Aliases { get; set; } = new();
|
||||
|
||||
[BsonElement("purls")]
|
||||
public List<string> Purls { get; set; } = new();
|
||||
|
||||
[BsonElement("cpes")]
|
||||
public List<string> Cpes { get; set; } = new();
|
||||
|
||||
[BsonElement("scopes")]
|
||||
public List<string> Scopes { get; set; } = new();
|
||||
|
||||
[BsonElement("relationships")]
|
||||
public List<AdvisoryObservationRelationshipDocument> Relationships { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class AdvisoryObservationRelationshipDocument
|
||||
{
|
||||
[BsonElement("type")]
|
||||
public string Type { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("source")]
|
||||
public string Source { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("target")]
|
||||
public string Target { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("provenance")]
|
||||
public string? Provenance { get; set; }
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Concelier.Core.Observations;
|
||||
@@ -9,14 +10,37 @@ namespace StellaOps.Concelier.Storage.Mongo.Observations;
|
||||
internal sealed class AdvisoryObservationSink : IAdvisoryObservationSink
|
||||
{
|
||||
private readonly IAdvisoryObservationStore _store;
|
||||
private readonly IAdvisoryObservationEventPublisher _publisher;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public AdvisoryObservationSink(IAdvisoryObservationStore store)
|
||||
public AdvisoryObservationSink(
|
||||
IAdvisoryObservationStore store,
|
||||
IAdvisoryObservationEventPublisher publisher,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(AdvisoryObservation observation, CancellationToken cancellationToken)
|
||||
{
|
||||
return _store.UpsertAsync(observation, cancellationToken);
|
||||
ArgumentNullException.ThrowIfNull(observation);
|
||||
|
||||
return UpsertAndPublishAsync(observation, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task UpsertAndPublishAsync(AdvisoryObservation observation, CancellationToken cancellationToken)
|
||||
{
|
||||
await _store.UpsertAsync(observation, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var evt = AdvisoryObservationUpdatedEvent.FromObservation(
|
||||
observation,
|
||||
supersedesId: observation.Attributes.GetValueOrDefault("supersedesId")
|
||||
?? observation.Attributes.GetValueOrDefault("supersedes"),
|
||||
traceId: observation.Attributes.GetValueOrDefault("traceId"),
|
||||
replayCursor: _timeProvider.GetUtcNow().Ticks.ToString());
|
||||
|
||||
await _publisher.PublishAsync(evt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Concelier.Core.Observations;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Mongo.Observations;
|
||||
|
||||
internal sealed class AdvisoryObservationTransportWorker : BackgroundService
|
||||
{
|
||||
private readonly IAdvisoryObservationEventOutbox _outbox;
|
||||
private readonly IAdvisoryObservationEventPublisher _publisher;
|
||||
private readonly ILogger<AdvisoryObservationTransportWorker> _logger;
|
||||
private readonly AdvisoryObservationEventPublisherOptions _options;
|
||||
|
||||
public AdvisoryObservationTransportWorker(
|
||||
IAdvisoryObservationEventOutbox outbox,
|
||||
IAdvisoryObservationEventPublisher publisher,
|
||||
IOptions<AdvisoryObservationEventPublisherOptions> options,
|
||||
ILogger<AdvisoryObservationTransportWorker> logger)
|
||||
{
|
||||
_outbox = outbox ?? throw new ArgumentNullException(nameof(outbox));
|
||||
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = options.Value;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!_options.Enabled)
|
||||
{
|
||||
_logger.LogInformation("Observation transport worker disabled.");
|
||||
return;
|
||||
}
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var batch = await _outbox.DequeueAsync(25, stoppingToken).ConfigureAwait(false);
|
||||
if (batch.Count == 0)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var evt in batch)
|
||||
{
|
||||
await _publisher.PublishAsync(evt, stoppingToken).ConfigureAwait(false);
|
||||
await _outbox.MarkPublishedAsync(evt.EventId, DateTimeOffset.UtcNow, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Observation transport worker error; retrying");
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Core.Observations;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Mongo.Observations;
|
||||
|
||||
internal sealed class MongoAdvisoryObservationEventOutbox : IAdvisoryObservationEventOutbox
|
||||
{
|
||||
private readonly IMongoCollection<AdvisoryObservationEventDocument> _collection;
|
||||
|
||||
public MongoAdvisoryObservationEventOutbox(IMongoCollection<AdvisoryObservationEventDocument> collection)
|
||||
{
|
||||
_collection = collection ?? throw new ArgumentNullException(nameof(collection));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyCollection<AdvisoryObservationUpdatedEvent>> DequeueAsync(int take, CancellationToken cancellationToken)
|
||||
{
|
||||
if (take <= 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryObservationUpdatedEvent>();
|
||||
}
|
||||
|
||||
var filter = Builders<AdvisoryObservationEventDocument>.Filter.Eq(doc => doc.PublishedAt, null);
|
||||
var documents = await _collection
|
||||
.Find(filter)
|
||||
.SortByDescending(doc => doc.IngestedAt)
|
||||
.Limit(take)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents.Select(ToDomain).ToArray();
|
||||
}
|
||||
|
||||
public Task MarkPublishedAsync(Guid eventId, DateTimeOffset publishedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
var update = Builders<AdvisoryObservationEventDocument>.Update.Set(doc => doc.PublishedAt, publishedAt.UtcDateTime);
|
||||
return _collection.UpdateOneAsync(
|
||||
Builders<AdvisoryObservationEventDocument>.Filter.Eq(doc => doc.Id, eventId),
|
||||
update,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
private static AdvisoryObservationUpdatedEvent ToDomain(AdvisoryObservationEventDocument doc)
|
||||
{
|
||||
return new AdvisoryObservationUpdatedEvent(
|
||||
doc.Id,
|
||||
doc.TenantId,
|
||||
doc.ObservationId,
|
||||
doc.AdvisoryId,
|
||||
new Models.Observations.AdvisoryObservationSource(
|
||||
doc.Source.Vendor,
|
||||
doc.Source.Stream,
|
||||
doc.Source.Api,
|
||||
doc.Source.CollectorVersion),
|
||||
new AdvisoryObservationLinksetSummary(
|
||||
doc.LinksetSummary.Aliases.ToImmutableArray(),
|
||||
doc.LinksetSummary.Purls.ToImmutableArray(),
|
||||
doc.LinksetSummary.Cpes.ToImmutableArray(),
|
||||
doc.LinksetSummary.Scopes.ToImmutableArray(),
|
||||
doc.LinksetSummary.Relationships
|
||||
.Select(rel => new AdvisoryObservationRelationshipSummary(rel.Type, rel.Source, rel.Target, rel.Provenance))
|
||||
.ToImmutableArray()),
|
||||
doc.DocumentSha,
|
||||
doc.ObservationHash,
|
||||
doc.IngestedAt,
|
||||
doc.ReplayCursor,
|
||||
doc.SupersedesId,
|
||||
doc.TraceId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Core.Observations;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Mongo.Observations;
|
||||
|
||||
internal sealed class MongoAdvisoryObservationEventPublisher : IAdvisoryObservationEventPublisher
|
||||
{
|
||||
private readonly IMongoCollection<AdvisoryObservationEventDocument> _collection;
|
||||
|
||||
public MongoAdvisoryObservationEventPublisher(IMongoCollection<AdvisoryObservationEventDocument> collection)
|
||||
{
|
||||
_collection = collection ?? throw new ArgumentNullException(nameof(collection));
|
||||
}
|
||||
|
||||
public Task PublishAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
|
||||
var document = new AdvisoryObservationEventDocument
|
||||
{
|
||||
Id = @event.EventId,
|
||||
TenantId = @event.TenantId,
|
||||
ObservationId = @event.ObservationId,
|
||||
AdvisoryId = @event.AdvisoryId,
|
||||
DocumentSha = @event.DocumentSha,
|
||||
ObservationHash = @event.ObservationHash,
|
||||
IngestedAt = @event.IngestedAt.UtcDateTime,
|
||||
ReplayCursor = @event.ReplayCursor,
|
||||
SupersedesId = @event.SupersedesId,
|
||||
TraceId = @event.TraceId,
|
||||
Source = new AdvisoryObservationSourceDocument
|
||||
{
|
||||
Vendor = @event.Source.Vendor,
|
||||
Stream = @event.Source.Stream,
|
||||
Api = @event.Source.Api,
|
||||
CollectorVersion = @event.Source.CollectorVersion
|
||||
},
|
||||
LinksetSummary = new AdvisoryObservationLinksetSummaryDocument
|
||||
{
|
||||
Aliases = @event.LinksetSummary.Aliases.ToList(),
|
||||
Purls = @event.LinksetSummary.Purls.ToList(),
|
||||
Cpes = @event.LinksetSummary.Cpes.ToList(),
|
||||
Scopes = @event.LinksetSummary.Scopes.ToList(),
|
||||
Relationships = @event.LinksetSummary.Relationships
|
||||
.Select(static rel => new AdvisoryObservationRelationshipDocument
|
||||
{
|
||||
Type = rel.Type,
|
||||
Source = rel.Source,
|
||||
Target = rel.Target,
|
||||
Provenance = rel.Provenance
|
||||
})
|
||||
.ToList()
|
||||
}
|
||||
};
|
||||
|
||||
return _collection.InsertOneAsync(document, cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NATS.Client.Core;
|
||||
using NATS.Client.JetStream;
|
||||
using StellaOps.Concelier.Core.Observations;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Mongo.Observations;
|
||||
|
||||
internal sealed class NatsAdvisoryObservationEventPublisher : IAdvisoryObservationEventPublisher
|
||||
{
|
||||
private readonly ILogger<NatsAdvisoryObservationEventPublisher> _logger;
|
||||
private readonly AdvisoryObservationEventPublisherOptions _options;
|
||||
|
||||
public NatsAdvisoryObservationEventPublisher(
|
||||
IOptions<AdvisoryObservationEventPublisherOptions> options,
|
||||
ILogger<NatsAdvisoryObservationEventPublisher> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = options.Value;
|
||||
}
|
||||
|
||||
public async Task PublishAsync(AdvisoryObservationUpdatedEvent @event, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!_options.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var subject = _options.Subject;
|
||||
var payload = JsonSerializer.SerializeToUtf8Bytes(@event);
|
||||
var opts = new NatsOpts { Url = _options.NatsUrl ?? "nats://127.0.0.1:4222" };
|
||||
|
||||
await using var connection = new NatsConnection(opts);
|
||||
var js = new NatsJSContext(connection);
|
||||
|
||||
await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false);
|
||||
await js.PublishAsync(subject, payload, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogDebug("Published advisory.observation.updated@1 to NATS subject {Subject} for observation {ObservationId}", subject, @event.ObservationId);
|
||||
}
|
||||
|
||||
private async Task EnsureStreamAsync(INatsJSContext js, CancellationToken cancellationToken)
|
||||
{
|
||||
var stream = _options.Stream;
|
||||
try
|
||||
{
|
||||
await js.GetStreamAsync(stream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (NatsJSApiException ex) when (ex.Error?.Code == 404)
|
||||
{
|
||||
var cfg = new NatsJSStreamConfig
|
||||
{
|
||||
Name = stream,
|
||||
Subjects = new[] { _options.Subject },
|
||||
Description = "Concelier advisory observation events",
|
||||
MaxMsgSize = 512 * 1024,
|
||||
};
|
||||
await js.CreateStreamAsync(cfg, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,19 @@ public static class ServiceCollectionExtensions
|
||||
services.AddSingleton<IAdvisoryObservationLookup, AdvisoryObservationLookup>();
|
||||
services.AddSingleton<IAdvisoryEventRepository, MongoAdvisoryEventRepository>();
|
||||
services.AddSingleton<IAdvisoryEventLog, AdvisoryEventLog>();
|
||||
services.AddSingleton<MongoAdvisoryObservationEventPublisher>();
|
||||
services.AddSingleton<NatsAdvisoryObservationEventPublisher>();
|
||||
services.AddSingleton<IAdvisoryObservationEventPublisher>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<AdvisoryObservationEventPublisherOptions>>().Value;
|
||||
if (string.Equals(options.Transport, "nats", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return sp.GetRequiredService<NatsAdvisoryObservationEventPublisher>();
|
||||
}
|
||||
|
||||
return sp.GetRequiredService<MongoAdvisoryObservationEventPublisher>();
|
||||
});
|
||||
services.AddSingleton<IAdvisoryObservationEventOutbox, MongoAdvisoryObservationEventOutbox>();
|
||||
services.AddSingleton<IAdvisoryRawRepository, MongoAdvisoryRawRepository>();
|
||||
services.AddSingleton<StellaOps.Concelier.Storage.Mongo.Linksets.IMongoAdvisoryLinksetStore, StellaOps.Concelier.Storage.Mongo.Linksets.ConcelierMongoLinksetStore>();
|
||||
services.AddSingleton<StellaOps.Concelier.Core.Linksets.IAdvisoryLinksetStore>(sp =>
|
||||
@@ -108,6 +121,12 @@ public static class ServiceCollectionExtensions
|
||||
return database.GetCollection<AdvisoryObservationDocument>(MongoStorageDefaults.Collections.AdvisoryObservations);
|
||||
});
|
||||
|
||||
services.AddSingleton<IMongoCollection<AdvisoryObservationEventDocument>>(static sp =>
|
||||
{
|
||||
var database = sp.GetRequiredService<IMongoDatabase>();
|
||||
return database.GetCollection<AdvisoryObservationEventDocument>(MongoStorageDefaults.Collections.AdvisoryObservationEvents);
|
||||
});
|
||||
|
||||
services.AddSingleton<IMongoCollection<AdvisoryLinksetDocument>>(static sp =>
|
||||
{
|
||||
var database = sp.GetRequiredService<IMongoDatabase>();
|
||||
@@ -126,8 +145,11 @@ public static class ServiceCollectionExtensions
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryObservationsRawLinksetMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryLinksetsTenantLowerMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryEventCollectionsMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryObservationEventCollectionMigration>();
|
||||
services.AddSingleton<IMongoMigration, SemVerStyleBackfillMigration>();
|
||||
|
||||
services.AddSingleton<IHostedService, AdvisoryObservationTransportWorker>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,12 @@
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="NATS.Client.Core" Version="2.0.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
using System.Collections.Immutable;
|
||||
using StellaOps.Concelier.Core.Linksets;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Tests.Linksets;
|
||||
|
||||
public sealed class AdvisoryLinksetNormalizationConfidenceTests
|
||||
{
|
||||
[Fact]
|
||||
public void FromRawLinksetWithConfidence_ComputesWeightedScoreAndReasons()
|
||||
{
|
||||
var linkset = new RawLinkset
|
||||
{
|
||||
Aliases = ImmutableArray.Create("CVE-2024-11111", "GHSA-aaaa-bbbb"),
|
||||
PackageUrls = ImmutableArray.Create("pkg:npm/foo@1.0.0", "pkg:npm/foo@1.1.0"),
|
||||
Cpes = ImmutableArray.Create("cpe:/a:foo:foo:1.0.0", "cpe:/a:foo:foo:1.1.0"),
|
||||
Notes = ImmutableDictionary.CreateRange(new[] { new KeyValuePair<string, string>("severity", "mismatch") })
|
||||
};
|
||||
|
||||
var (normalized, confidence, conflicts) = AdvisoryLinksetNormalization.FromRawLinksetWithConfidence(linkset);
|
||||
|
||||
Assert.NotNull(normalized);
|
||||
Assert.NotNull(confidence);
|
||||
Assert.True(confidence!.Value is > 0.7 and < 0.8); // weighted score with conflict penalty
|
||||
|
||||
var conflict = Assert.Single(conflicts);
|
||||
Assert.Equal("severity-mismatch", conflict.Reason);
|
||||
Assert.Contains("severity:mismatch", conflict.Values!);
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,7 @@ public sealed class AdvisoryObservationAggregationTests
|
||||
var confidence = result.confidence;
|
||||
var conflicts = result.conflicts;
|
||||
|
||||
Assert.Equal(0.5, confidence);
|
||||
Assert.True(confidence is >= 0.1 and <= 0.6);
|
||||
Assert.Single(conflicts);
|
||||
Assert.Null(normalized); // no purls supplied
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using StellaOps.Concelier.Core.Observations;
|
||||
using StellaOps.Concelier.Models.Observations;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Tests.Observations;
|
||||
|
||||
public sealed class AdvisoryObservationEventFactoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void FromObservation_MapsFieldsAndHashesDeterministically()
|
||||
{
|
||||
var observation = CreateObservation();
|
||||
|
||||
var evt = AdvisoryObservationUpdatedEvent.FromObservation(
|
||||
observation,
|
||||
supersedesId: "655fabcdedc0ffee0000abcd",
|
||||
traceId: "trace-123");
|
||||
|
||||
Assert.Equal("urn:tenant:tenant-1", evt.TenantId);
|
||||
Assert.Equal("adv-1", evt.AdvisoryId);
|
||||
Assert.Equal("655fabcdedc0ffee0000abcd", evt.SupersedesId);
|
||||
Assert.NotNull(evt.ObservationHash);
|
||||
Assert.Equal(observation.Upstream.ContentHash, evt.DocumentSha);
|
||||
Assert.Contains("pkg:npm/foo", evt.LinksetSummary.Purls);
|
||||
}
|
||||
|
||||
private static AdvisoryObservation CreateObservation()
|
||||
{
|
||||
var source = new AdvisoryObservationSource("ghsa", "advisories", "https://api");
|
||||
var upstream = new AdvisoryObservationUpstream(
|
||||
"adv-1",
|
||||
"v1",
|
||||
DateTimeOffset.Parse("2025-11-20T12:00:00Z"),
|
||||
DateTimeOffset.Parse("2025-11-20T12:00:00Z"),
|
||||
"2f8f568cc1ed3474f0a4564ddb8c64f4b4d176fbe0a2a98a02b88e822a4f5b6d",
|
||||
new AdvisoryObservationSignature(false, null, null, null));
|
||||
|
||||
var content = new AdvisoryObservationContent("json", null, JsonNode.Parse("{}")!);
|
||||
var linkset = new AdvisoryObservationLinkset(
|
||||
aliases: new[] { "CVE-2024-1234", "GHSA-xxxx" },
|
||||
purls: new[] { "pkg:npm/foo@1.0.0" },
|
||||
cpes: new[] { "cpe:/a:foo:foo:1.0.0" },
|
||||
references: new[] { new AdvisoryObservationReference("ref", "https://example.com") });
|
||||
|
||||
var rawLinkset = new RawLinkset
|
||||
{
|
||||
Aliases = ImmutableArray.Create("CVE-2024-1234", "GHSA-xxxx"),
|
||||
PackageUrls = ImmutableArray.Create("pkg:npm/foo@1.0.0"),
|
||||
Cpes = ImmutableArray.Create("cpe:/a:foo:foo:1.0.0"),
|
||||
Scopes = ImmutableArray.Create("runtime"),
|
||||
Relationships = ImmutableArray.Create(new RawRelationship("contains", "pkg:npm/foo@1.0.0", "file://dist/foo.js")),
|
||||
};
|
||||
|
||||
return new AdvisoryObservation(
|
||||
"655fabcdf3c5d6ad3b5a0aaa",
|
||||
"tenant-1",
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
linkset,
|
||||
rawLinkset,
|
||||
DateTimeOffset.Parse("2025-11-20T12:01:00Z"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<DefaultItemExcludes>$(DefaultItemExcludes);**/tools/**/*</DefaultItemExcludes>
|
||||
<DisableTransitiveProjectReferences>true</DisableTransitiveProjectReferences>
|
||||
<MSBuildProjectExtensionsPath>$(MSBuildThisFileDirectory)obj/$(MSBuildProjectName)/</MSBuildProjectExtensionsPath>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Compile Remove="**/tools/**/*.cs" />
|
||||
<None Remove="**/tools/**/*" />
|
||||
<None Include="**/tools/**/*" Pack="false" CopyToOutputDirectory="Never" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
|
||||
<PackageReference Include="xunit" Version="2.8.1" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.1">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<FrameworkReference Include="Microsoft.AspNetCore.App" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="StellaOps.Findings.Ledger">
|
||||
<HintPath>..\StellaOps.Findings.Ledger\bin\Release\net10.0\StellaOps.Findings.Ledger.dll</HintPath>
|
||||
<Private>true</Private>
|
||||
</Reference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Remove="**/*.cs" />
|
||||
<Compile Include="Exports/ExportPagingTests.cs" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,72 @@
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace StellaOps.Findings.Ledger.WebService.Contracts;
|
||||
|
||||
public sealed record ExportFindingsRequest(
|
||||
string TenantId,
|
||||
string Shape,
|
||||
long? SinceSequence,
|
||||
long? UntilSequence,
|
||||
DateTimeOffset? SinceObservedAt,
|
||||
DateTimeOffset? UntilObservedAt,
|
||||
string? Status,
|
||||
decimal? Severity,
|
||||
int PageSize,
|
||||
string FiltersHash,
|
||||
ExportPagingKey? PagingKey);
|
||||
|
||||
public sealed record ExportPagingKey(long SequenceNumber, string PolicyVersion, string CycleHash);
|
||||
|
||||
public sealed record FindingExportItem(
|
||||
long EventSequence,
|
||||
DateTimeOffset ObservedAt,
|
||||
string FindingId,
|
||||
string PolicyVersion,
|
||||
string Status,
|
||||
decimal? Severity,
|
||||
string CycleHash,
|
||||
string? EvidenceBundleRef,
|
||||
ExportProvenance Provenance,
|
||||
JsonObject? Labels);
|
||||
|
||||
public sealed record VexExportItem(
|
||||
long EventSequence,
|
||||
DateTimeOffset ObservedAt,
|
||||
string VexStatementId,
|
||||
string ProductId,
|
||||
string Status,
|
||||
string? StatementType,
|
||||
bool? KnownExploited,
|
||||
string CycleHash,
|
||||
ExportProvenance Provenance);
|
||||
|
||||
public sealed record AdvisoryExportItem(
|
||||
long EventSequence,
|
||||
DateTimeOffset Published,
|
||||
string AdvisoryId,
|
||||
string Source,
|
||||
string Title,
|
||||
string? Severity,
|
||||
decimal? CvssScore,
|
||||
string? CvssVector,
|
||||
bool? Kev,
|
||||
string CycleHash,
|
||||
ExportProvenance Provenance);
|
||||
|
||||
public sealed record SbomExportItem(
|
||||
long EventSequence,
|
||||
DateTimeOffset CreatedAt,
|
||||
string SbomId,
|
||||
string SubjectDigest,
|
||||
string SbomFormat,
|
||||
int ComponentsCount,
|
||||
bool? HasVulnerabilities,
|
||||
string CycleHash,
|
||||
ExportProvenance Provenance);
|
||||
|
||||
public sealed record ExportProvenance(
|
||||
string PolicyVersion,
|
||||
string CycleHash,
|
||||
string? LedgerEventHash);
|
||||
|
||||
public sealed record ExportPage<T>(IReadOnlyList<T> Items, string? NextPageToken);
|
||||
@@ -1,6 +1,8 @@
|
||||
using Microsoft.AspNetCore.Diagnostics;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Serilog;
|
||||
using Serilog.Events;
|
||||
@@ -14,14 +16,17 @@ using StellaOps.Findings.Ledger.Infrastructure.Merkle;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Postgres;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Projection;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Policy;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Exports;
|
||||
using StellaOps.Findings.Ledger.Options;
|
||||
using StellaOps.Findings.Ledger.Services;
|
||||
using StellaOps.Findings.Ledger.WebService.Contracts;
|
||||
using StellaOps.Findings.Ledger.WebService.Mappings;
|
||||
using StellaOps.Findings.Ledger.WebService.Services;
|
||||
using StellaOps.Telemetry.Core;
|
||||
using StellaOps.Findings.Ledger.Services.Security;
|
||||
|
||||
const string LedgerWritePolicy = "ledger.events.write";
|
||||
const string LedgerExportPolicy = "ledger.export.read";
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
@@ -112,6 +117,13 @@ builder.Services.AddAuthorization(options =>
|
||||
policy.Requirements.Add(new StellaOpsScopeRequirement(scopes));
|
||||
policy.AddAuthenticationSchemes(StellaOpsAuthenticationDefaults.AuthenticationScheme);
|
||||
});
|
||||
|
||||
options.AddPolicy(LedgerExportPolicy, policy =>
|
||||
{
|
||||
policy.RequireAuthenticatedUser();
|
||||
policy.Requirements.Add(new StellaOpsScopeRequirement(scopes));
|
||||
policy.AddAuthenticationSchemes(StellaOpsAuthenticationDefaults.AuthenticationScheme);
|
||||
});
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<LedgerAnchorQueue>();
|
||||
@@ -133,6 +145,7 @@ builder.Services.AddSingleton<IAttachmentUrlSigner, AttachmentUrlSigner>();
|
||||
builder.Services.AddSingleton<IConsoleCsrfValidator, ConsoleCsrfValidator>();
|
||||
builder.Services.AddHostedService<LedgerMerkleAnchorWorker>();
|
||||
builder.Services.AddHostedService<LedgerProjectionWorker>();
|
||||
builder.Services.AddSingleton<ExportQueryService>();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
@@ -197,6 +210,118 @@ app.MapPost("/vuln/ledger/events", async Task<Results<Created<LedgerEventRespons
|
||||
.ProducesProblem(StatusCodes.Status409Conflict)
|
||||
.ProducesProblem(StatusCodes.Status500InternalServerError);
|
||||
|
||||
app.MapGet("/ledger/export/findings", async Task<Results<FileStreamHttpResult, JsonHttpResult<ExportPage<FindingExportItem>>, ProblemHttpResult>> (
|
||||
HttpContext httpContext,
|
||||
ExportQueryService exportQueryService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!httpContext.Request.Headers.TryGetValue("X-Stella-Tenant", out var tenantValues) || string.IsNullOrWhiteSpace(tenantValues))
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status400BadRequest, title: "missing_tenant", detail: "X-Stella-Tenant header is required.");
|
||||
}
|
||||
|
||||
var tenantId = tenantValues.ToString();
|
||||
var shape = httpContext.Request.Query["shape"].ToString();
|
||||
if (string.IsNullOrWhiteSpace(shape))
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status400BadRequest, title: "missing_shape", detail: "shape is required (canonical|compact).");
|
||||
}
|
||||
|
||||
var pageSize = exportQueryService.ClampPageSize(ParseInt(httpContext.Request.Query["page_size"]));
|
||||
|
||||
long? sinceSequence = ParseLong(httpContext.Request.Query["since_sequence"]);
|
||||
long? untilSequence = ParseLong(httpContext.Request.Query["until_sequence"]);
|
||||
DateTimeOffset? sinceObservedAt = ParseDate(httpContext.Request.Query["since_observed_at"]);
|
||||
DateTimeOffset? untilObservedAt = ParseDate(httpContext.Request.Query["until_observed_at"]);
|
||||
var status = httpContext.Request.Query["finding_status"].ToString();
|
||||
var severity = ParseDecimal(httpContext.Request.Query["severity"]);
|
||||
|
||||
var request = new ExportFindingsRequest(
|
||||
TenantId: tenantId,
|
||||
Shape: shape,
|
||||
SinceSequence: sinceSequence,
|
||||
UntilSequence: untilSequence,
|
||||
SinceObservedAt: sinceObservedAt,
|
||||
UntilObservedAt: untilObservedAt,
|
||||
Status: string.IsNullOrWhiteSpace(status) ? null : status,
|
||||
Severity: severity,
|
||||
PageSize: pageSize,
|
||||
FiltersHash: string.Empty,
|
||||
PagingKey: null);
|
||||
|
||||
var filtersHash = exportQueryService.ComputeFiltersHash(request);
|
||||
|
||||
ExportPagingKey? pagingKey = null;
|
||||
var pageToken = httpContext.Request.Query["page_token"].ToString();
|
||||
if (!string.IsNullOrWhiteSpace(pageToken))
|
||||
{
|
||||
if (!ExportPaging.TryParsePageToken(pageToken, filtersHash, out var parsedKey, out var error))
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status400BadRequest, title: error ?? "invalid_page_token");
|
||||
}
|
||||
|
||||
pagingKey = new ExportPagingKey(parsedKey!.SequenceNumber, parsedKey.PolicyVersion, parsedKey.CycleHash);
|
||||
}
|
||||
|
||||
request = request with { FiltersHash = filtersHash, PagingKey = pagingKey };
|
||||
|
||||
ExportPage<FindingExportItem> page;
|
||||
try
|
||||
{
|
||||
page = await exportQueryService.GetFindingsAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (InvalidOperationException ex) when (ex.Message == "filters_hash_mismatch")
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status400BadRequest, title: "page_token_filters_mismatch");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(page.NextPageToken))
|
||||
{
|
||||
httpContext.Response.Headers["X-Stella-Next-Page-Token"] = page.NextPageToken;
|
||||
}
|
||||
httpContext.Response.Headers["X-Stella-Result-Count"] = page.Items.Count.ToString();
|
||||
|
||||
var acceptsNdjson = httpContext.Request.Headers.Accept.Any(h => h.Contains("application/x-ndjson", StringComparison.OrdinalIgnoreCase));
|
||||
if (acceptsNdjson)
|
||||
{
|
||||
httpContext.Response.ContentType = "application/x-ndjson";
|
||||
var stream = new MemoryStream();
|
||||
await using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { SkipValidation = false, Indented = false });
|
||||
foreach (var item in page.Items)
|
||||
{
|
||||
JsonSerializer.Serialize(writer, item);
|
||||
writer.Flush();
|
||||
await stream.WriteAsync(new byte[] { (byte)'\n' }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
stream.Position = 0;
|
||||
return TypedResults.Stream(stream, contentType: "application/x-ndjson");
|
||||
}
|
||||
|
||||
return TypedResults.Json(page);
|
||||
})
|
||||
.WithName("LedgerExportFindings")
|
||||
.RequireAuthorization(LedgerExportPolicy)
|
||||
.Produces(StatusCodes.Status200OK)
|
||||
.ProducesProblem(StatusCodes.Status400BadRequest)
|
||||
.ProducesProblem(StatusCodes.Status401Unauthorized)
|
||||
.ProducesProblem(StatusCodes.Status403Forbidden)
|
||||
.ProducesProblem(StatusCodes.Status500InternalServerError);
|
||||
|
||||
app.MapGet("/ledger/export/vex", () => TypedResults.Json(new ExportPage<VexExportItem>(Array.Empty<VexExportItem>(), null)))
|
||||
.WithName("LedgerExportVex")
|
||||
.RequireAuthorization(LedgerExportPolicy)
|
||||
.Produces(StatusCodes.Status200OK);
|
||||
|
||||
app.MapGet("/ledger/export/advisories", () => TypedResults.Json(new ExportPage<AdvisoryExportItem>(Array.Empty<AdvisoryExportItem>(), null)))
|
||||
.WithName("LedgerExportAdvisories")
|
||||
.RequireAuthorization(LedgerExportPolicy)
|
||||
.Produces(StatusCodes.Status200OK);
|
||||
|
||||
app.MapGet("/ledger/export/sboms", () => TypedResults.Json(new ExportPage<SbomExportItem>(Array.Empty<SbomExportItem>(), null)))
|
||||
.WithName("LedgerExportSboms")
|
||||
.RequireAuthorization(LedgerExportPolicy)
|
||||
.Produces(StatusCodes.Status200OK);
|
||||
|
||||
app.Run();
|
||||
|
||||
static Created<LedgerEventResponse> CreateCreatedResponse(LedgerEventRecord record)
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using NpgsqlTypes;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Exports;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Postgres;
|
||||
using StellaOps.Findings.Ledger.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Findings.Ledger.WebService.Services;
|
||||
|
||||
public sealed class ExportQueryService
|
||||
{
|
||||
private const int DefaultPageSize = 500;
|
||||
private const int MaxPageSize = 5000;
|
||||
|
||||
private readonly LedgerDataSource _dataSource;
|
||||
private readonly ILogger<ExportQueryService> _logger;
|
||||
|
||||
public ExportQueryService(LedgerDataSource dataSource, ILogger<ExportQueryService> logger)
|
||||
{
|
||||
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public ExportPage<VexExportItem> GetVexEmpty() => new(Array.Empty<VexExportItem>(), null);
|
||||
|
||||
public ExportPage<AdvisoryExportItem> GetAdvisoriesEmpty() => new(Array.Empty<AdvisoryExportItem>(), null);
|
||||
|
||||
public ExportPage<SbomExportItem> GetSbomsEmpty() => new(Array.Empty<SbomExportItem>(), null);
|
||||
|
||||
public int ClampPageSize(int? requested)
|
||||
{
|
||||
if (!requested.HasValue || requested.Value <= 0)
|
||||
{
|
||||
return DefaultPageSize;
|
||||
}
|
||||
|
||||
return Math.Min(requested.Value, MaxPageSize);
|
||||
}
|
||||
|
||||
public string ComputeFiltersHash(ExportFindingsRequest request)
|
||||
{
|
||||
var filters = new Dictionary<string, string?>
|
||||
{
|
||||
["shape"] = request.Shape,
|
||||
["since_sequence"] = request.SinceSequence?.ToString(),
|
||||
["until_sequence"] = request.UntilSequence?.ToString(),
|
||||
["since_observed_at"] = request.SinceObservedAt?.ToString("O"),
|
||||
["until_observed_at"] = request.UntilObservedAt?.ToString("O"),
|
||||
["status"] = request.Status,
|
||||
["severity"] = request.Severity?.ToString()
|
||||
};
|
||||
|
||||
return ExportPaging.ComputeFiltersHash(filters);
|
||||
}
|
||||
|
||||
public async Task<ExportPage<FindingExportItem>> GetFindingsAsync(ExportFindingsRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (!string.Equals(request.FiltersHash, ComputeFiltersHash(request), StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("filters_hash_mismatch");
|
||||
}
|
||||
|
||||
const string baseSql = """
|
||||
SELECT le.sequence_no,
|
||||
le.recorded_at,
|
||||
fp.finding_id,
|
||||
fp.policy_version,
|
||||
fp.status,
|
||||
fp.severity,
|
||||
fp.labels,
|
||||
fp.cycle_hash,
|
||||
le.evidence_bundle_ref,
|
||||
le.event_hash
|
||||
FROM findings_projection fp
|
||||
JOIN ledger_events le
|
||||
ON le.tenant_id = fp.tenant_id
|
||||
AND le.event_id = fp.current_event_id
|
||||
WHERE fp.tenant_id = @tenant_id
|
||||
""";
|
||||
|
||||
var sqlBuilder = new System.Text.StringBuilder(baseSql);
|
||||
var parameters = new List<NpgsqlParameter>
|
||||
{
|
||||
new("tenant_id", request.TenantId)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Text
|
||||
}
|
||||
};
|
||||
|
||||
if (request.SinceSequence.HasValue)
|
||||
{
|
||||
sqlBuilder.Append(" AND le.sequence_no >= @since_sequence");
|
||||
parameters.Add(new NpgsqlParameter<long>("since_sequence", request.SinceSequence.Value)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Bigint
|
||||
});
|
||||
}
|
||||
|
||||
if (request.UntilSequence.HasValue)
|
||||
{
|
||||
sqlBuilder.Append(" AND le.sequence_no <= @until_sequence");
|
||||
parameters.Add(new NpgsqlParameter<long>("until_sequence", request.UntilSequence.Value)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Bigint
|
||||
});
|
||||
}
|
||||
|
||||
if (request.SinceObservedAt.HasValue)
|
||||
{
|
||||
sqlBuilder.Append(" AND le.recorded_at >= @since_observed_at");
|
||||
parameters.Add(new NpgsqlParameter<DateTimeOffset>("since_observed_at", request.SinceObservedAt.Value)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.TimestampTz
|
||||
});
|
||||
}
|
||||
|
||||
if (request.UntilObservedAt.HasValue)
|
||||
{
|
||||
sqlBuilder.Append(" AND le.recorded_at <= @until_observed_at");
|
||||
parameters.Add(new NpgsqlParameter<DateTimeOffset>("until_observed_at", request.UntilObservedAt.Value)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.TimestampTz
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(request.Status))
|
||||
{
|
||||
sqlBuilder.Append(" AND fp.status = @status");
|
||||
parameters.Add(new NpgsqlParameter<string>("status", request.Status)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Text
|
||||
});
|
||||
}
|
||||
|
||||
if (request.Severity.HasValue)
|
||||
{
|
||||
sqlBuilder.Append(" AND fp.severity = @severity");
|
||||
parameters.Add(new NpgsqlParameter<decimal>("severity", request.Severity.Value)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Numeric
|
||||
});
|
||||
}
|
||||
|
||||
if (request.PagingKey is not null)
|
||||
{
|
||||
sqlBuilder.Append(" AND (le.sequence_no > @cursor_seq OR (le.sequence_no = @cursor_seq AND fp.policy_version > @cursor_policy) OR (le.sequence_no = @cursor_seq AND fp.policy_version = @cursor_policy AND fp.cycle_hash > @cursor_cycle))");
|
||||
parameters.Add(new NpgsqlParameter<long>("cursor_seq", request.PagingKey.SequenceNumber)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Bigint
|
||||
});
|
||||
parameters.Add(new NpgsqlParameter<string>("cursor_policy", request.PagingKey.PolicyVersion)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Text
|
||||
});
|
||||
parameters.Add(new NpgsqlParameter<string>("cursor_cycle", request.PagingKey.CycleHash)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Char
|
||||
});
|
||||
}
|
||||
|
||||
sqlBuilder.Append(" ORDER BY le.sequence_no, fp.policy_version, fp.cycle_hash");
|
||||
sqlBuilder.Append(" LIMIT @take");
|
||||
parameters.Add(new NpgsqlParameter<int>("take", request.PageSize + 1)
|
||||
{
|
||||
NpgsqlDbType = NpgsqlDbType.Integer
|
||||
});
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(request.TenantId, cancellationToken).ConfigureAwait(false);
|
||||
await using var command = new NpgsqlCommand(sqlBuilder.ToString(), connection)
|
||||
{
|
||||
CommandTimeout = _dataSource.CommandTimeoutSeconds
|
||||
};
|
||||
command.Parameters.AddRange(parameters.ToArray());
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
var items = new List<FindingExportItem>();
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var labelsJson = reader.GetFieldValue<string>(6);
|
||||
var labels = JsonNode.Parse(labelsJson)?.AsObject();
|
||||
|
||||
items.Add(new FindingExportItem(
|
||||
EventSequence: reader.GetInt64(0),
|
||||
ObservedAt: reader.GetFieldValue<DateTimeOffset>(1),
|
||||
FindingId: reader.GetString(2),
|
||||
PolicyVersion: reader.GetString(3),
|
||||
Status: reader.GetString(4),
|
||||
Severity: reader.IsDBNull(5) ? null : reader.GetDecimal(5),
|
||||
CycleHash: reader.GetString(7),
|
||||
EvidenceBundleRef: reader.IsDBNull(8) ? null : reader.GetString(8),
|
||||
Provenance: new ExportProvenance(
|
||||
PolicyVersion: reader.GetString(3),
|
||||
CycleHash: reader.GetString(7),
|
||||
LedgerEventHash: reader.IsDBNull(9) ? null : reader.GetString(9)),
|
||||
Labels: labels));
|
||||
}
|
||||
|
||||
string? nextPageToken = null;
|
||||
if (items.Count > request.PageSize)
|
||||
{
|
||||
var last = items[request.PageSize];
|
||||
items = items.Take(request.PageSize).ToList();
|
||||
var key = new ExportPagingKey(last.EventSequence, last.PolicyVersion, last.CycleHash);
|
||||
nextPageToken = ExportPaging.CreatePageToken(
|
||||
new ExportPaging.ExportPageKey(key.SequenceNumber, key.PolicyVersion, key.CycleHash),
|
||||
request.FiltersHash);
|
||||
}
|
||||
|
||||
return new ExportPage<FindingExportItem>(items, nextPageToken);
|
||||
}
|
||||
}
|
||||
16
src/Scanner/StellaOps.Scanner.Node.slnf
Normal file
16
src/Scanner/StellaOps.Scanner.Node.slnf
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"solution": {
|
||||
"path": "StellaOps.Scanner.sln",
|
||||
"projects": [
|
||||
"__Libraries/StellaOps.Scanner.Analyzers.Lang/StellaOps.Scanner.Analyzers.Lang.csproj",
|
||||
"__Libraries/StellaOps.Scanner.Analyzers.Lang.DotNet/StellaOps.Scanner.Analyzers.Lang.DotNet.csproj",
|
||||
"__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/StellaOps.Scanner.Analyzers.Lang.Node.csproj",
|
||||
"__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/StellaOps.Scanner.Analyzers.Lang.Ruby.csproj",
|
||||
"__Libraries/StellaOps.Scanner.Analyzers.Lang.Rust/StellaOps.Scanner.Analyzers.Lang.Rust.csproj",
|
||||
"__Libraries/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj",
|
||||
"__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj",
|
||||
"__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests.csproj",
|
||||
"../Concelier/__Libraries/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Core;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal.Phase22;
|
||||
|
||||
internal static class NodePhase22SampleLoader
|
||||
{
|
||||
private const string EnvKey = "SCANNER_NODE_PHASE22_FIXTURE";
|
||||
private const string DefaultFileName = "node-phase22-sample.ndjson";
|
||||
|
||||
public static async ValueTask<IReadOnlyCollection<LanguageComponentRecord>> TryLoadAsync(
|
||||
string rootPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var fixturePath = Environment.GetEnvironmentVariable(EnvKey);
|
||||
if (string.IsNullOrWhiteSpace(fixturePath))
|
||||
{
|
||||
fixturePath = Path.Combine(rootPath, DefaultFileName);
|
||||
if (!File.Exists(fixturePath))
|
||||
{
|
||||
// fallback to docs sample if tests point to repo root
|
||||
var repoRoot = FindRepoRoot(rootPath);
|
||||
var fromDocs = Path.Combine(repoRoot, "docs", "samples", "scanner", "node-phase22", DefaultFileName);
|
||||
fixturePath = File.Exists(fromDocs) ? fromDocs : fixturePath;
|
||||
}
|
||||
}
|
||||
|
||||
if (!File.Exists(fixturePath))
|
||||
{
|
||||
return Array.Empty<LanguageComponentRecord>();
|
||||
}
|
||||
|
||||
var records = new List<LanguageComponentRecord>();
|
||||
await using var stream = File.OpenRead(fixturePath);
|
||||
using var reader = new StreamReader(stream);
|
||||
|
||||
string? line;
|
||||
while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
using var jsonDoc = JsonDocument.Parse(line);
|
||||
var root = jsonDoc.RootElement;
|
||||
if (!root.TryGetProperty("type", out var typeProp))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.Equals(typeProp.GetString(), "component", StringComparison.Ordinal))
|
||||
{
|
||||
continue; // only components are mapped into LanguageComponentRecords for now
|
||||
}
|
||||
|
||||
var componentType = root.GetProperty("componentType").GetString() ?? "pkg";
|
||||
var path = root.GetProperty("path").GetString() ?? string.Empty;
|
||||
var reason = root.TryGetProperty("reason", out var reasonProp) ? reasonProp.GetString() : null;
|
||||
var format = root.TryGetProperty("format", out var formatProp) ? formatProp.GetString() : null;
|
||||
var confidence = root.TryGetProperty("confidence", out var confProp) && confProp.TryGetDouble(out var conf)
|
||||
? conf.ToString("0.00", CultureInfo.InvariantCulture)
|
||||
: null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = new List<KeyValuePair<string, string?>>();
|
||||
if (!string.IsNullOrWhiteSpace(reason)) metadata.Add(new("reason", reason));
|
||||
if (!string.IsNullOrWhiteSpace(format)) metadata.Add(new("format", format));
|
||||
if (!string.IsNullOrWhiteSpace(confidence)) metadata.Add(new("confidence", confidence));
|
||||
|
||||
var typeTag = componentType switch
|
||||
{
|
||||
"native" => "node:native",
|
||||
"wasm" => "node:wasm",
|
||||
_ => "node:bundle"
|
||||
};
|
||||
|
||||
var name = Path.GetFileName(path);
|
||||
var record = LanguageComponentRecord.FromExplicitKey(
|
||||
analyzerId: "node-phase22",
|
||||
componentKey: path,
|
||||
purl: null,
|
||||
name: name,
|
||||
version: null,
|
||||
type: typeTag,
|
||||
metadata: metadata,
|
||||
evidence: null,
|
||||
usedByEntrypoint: false);
|
||||
|
||||
records.Add(record);
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
private static string FindRepoRoot(string start)
|
||||
{
|
||||
var current = new DirectoryInfo(start);
|
||||
while (current is not null && current.Exists)
|
||||
{
|
||||
if (File.Exists(Path.Combine(current.FullName, "README.md")))
|
||||
{
|
||||
return current.FullName;
|
||||
}
|
||||
|
||||
current = current.Parent;
|
||||
}
|
||||
|
||||
return start;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang.Node.Internal;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Node;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Node.Internal;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Node.Internal.Phase22;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Node;
|
||||
|
||||
public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer
|
||||
{
|
||||
@@ -13,11 +14,11 @@ public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(writer);
|
||||
|
||||
var lockData = await NodeLockData.LoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
|
||||
var packages = NodePackageCollector.CollectPackages(context, lockData, cancellationToken);
|
||||
|
||||
foreach (var package in packages.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal))
|
||||
{
|
||||
var lockData = await NodeLockData.LoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
|
||||
var packages = NodePackageCollector.CollectPackages(context, lockData, cancellationToken);
|
||||
|
||||
foreach (var package in packages.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var metadata = package.CreateMetadata();
|
||||
@@ -29,9 +30,16 @@ public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer
|
||||
name: package.Name,
|
||||
version: package.Version,
|
||||
type: "npm",
|
||||
metadata: metadata,
|
||||
evidence: evidence,
|
||||
usedByEntrypoint: package.IsUsedByEntrypoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata: metadata,
|
||||
evidence: evidence,
|
||||
usedByEntrypoint: package.IsUsedByEntrypoint);
|
||||
}
|
||||
|
||||
// Optional Phase 22 prep path: ingest precomputed bundle/native/WASM AOC records from NDJSON fixture
|
||||
var phase22Records = await NodePhase22SampleLoader.TryLoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
|
||||
if (phase22Records.Count > 0)
|
||||
{
|
||||
writer.AddRange(phase22Records);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
{"type":"component","componentType":"pkg","path":"/src/app.js","format":"esm","fromBundle":true,"reason":"source-map","confidence":0.87,"resolverTrace":["bundle:/app/dist/main.js","map:/app/dist/main.js.map","source:/src/app.js"]}
|
||||
{"type":"component","componentType":"native","path":"/app/native/addon.node","arch":"x86_64","platform":"linux","reason":"native-addon-file","confidence":0.82,"resolverTrace":["file:/app/native/addon.node","require:/app/dist/native-entry.js"]}
|
||||
{"type":"component","componentType":"wasm","path":"/app/pkg/pkg.wasm","exports":["init","run"],"reason":"wasm-file","confidence":0.80,"resolverTrace":["file:/app/pkg/pkg.wasm","import:/app/dist/wasm-entry.js"]}
|
||||
@@ -0,0 +1,22 @@
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Node.Internal.Phase22;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Node.Tests;
|
||||
|
||||
public class NodePhase22SampleLoaderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task TryLoadAsync_ReadsComponentsFromNdjson()
|
||||
{
|
||||
var root = Path.Combine("Fixtures");
|
||||
var records = await NodePhase22SampleLoader.TryLoadAsync(root, CancellationToken.None);
|
||||
|
||||
Assert.Equal(3, records.Count);
|
||||
var native = records.Single(r => r.Type == "node:native");
|
||||
Assert.Equal("/app/native/addon.node", native.ComponentKey);
|
||||
}
|
||||
}
|
||||
15
src/Scanner/__Tests/node-isolated.runsettings
Normal file
15
src/Scanner/__Tests/node-isolated.runsettings
Normal file
@@ -0,0 +1,15 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<RunSettings>
|
||||
<RunConfiguration>
|
||||
<DisableAppDomain>true</DisableAppDomain>
|
||||
<MaxCpuCount>1</MaxCpuCount>
|
||||
<TargetPlatform>x64</TargetPlatform>
|
||||
<TargetFrameworkVersion>net10.0</TargetFrameworkVersion>
|
||||
<ResultsDirectory>./TestResults</ResultsDirectory>
|
||||
</RunConfiguration>
|
||||
<DataCollectionRunSettings>
|
||||
<DataCollectors>
|
||||
<!-- keep deterministic runs; no code coverage collectors by default -->
|
||||
</DataCollectors>
|
||||
</DataCollectionRunSettings>
|
||||
</RunSettings>
|
||||
22
src/Scanner/__Tests/node-tests-isolated.sh
Normal file
22
src/Scanner/__Tests/node-tests-isolated.sh
Normal file
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# repo root = two levels up from src/Scanner (__Tests/.. -> .. -> ..)
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
# Restore only filtered projects using offline/local feed
|
||||
NUGET_PACKAGES="$REPO_ROOT/offline/packages" \
|
||||
DOTNET_RESTORE_DISABLE_PARALLEL=true \
|
||||
DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 \
|
||||
dotnet restore src/Scanner/StellaOps.Scanner.Node.slnf \
|
||||
-p:RestorePackagesPath="$REPO_ROOT/offline/packages" \
|
||||
-p:ContinuousIntegrationBuild=true
|
||||
|
||||
# Run node analyzer tests in isolation
|
||||
DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 \
|
||||
DOTNET_CLI_TELEMETRY_OPTOUT=1 \
|
||||
dotnet test src/Scanner/StellaOps.Scanner.Node.slnf \
|
||||
--no-build \
|
||||
--settings "$REPO_ROOT/__Tests/node-isolated.runsettings" \
|
||||
/m:1
|
||||
Reference in New Issue
Block a user