up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Findings Ledger CI / build-test (push) Has been cancelled
Findings Ledger CI / migration-validation (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
cryptopro-linux-csp / build-and-test (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
sm-remote-ci / build-and-test (push) Has been cancelled
Findings Ledger CI / generate-manifest (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Findings Ledger CI / build-test (push) Has been cancelled
Findings Ledger CI / migration-validation (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
cryptopro-linux-csp / build-and-test (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
sm-remote-ci / build-and-test (push) Has been cancelled
Findings Ledger CI / generate-manifest (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
This commit is contained in:
@@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoContracts = StellaOps.Concelier.Storage.Mongo;
|
||||
using StorageContracts = StellaOps.Concelier.Storage.Contracts;
|
||||
using StellaOps.Concelier.Connector.Common.Http;
|
||||
using StellaOps.Concelier.Connector.Common.Telemetry;
|
||||
using StellaOps.Concelier.Core.Aoc;
|
||||
@@ -32,6 +33,7 @@ public sealed class SourceFetchService
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly MongoContracts.IDocumentStore _documentStore;
|
||||
private readonly StorageContracts.IStorageDocumentStore _storageDocumentStore;
|
||||
private readonly ILogger<SourceFetchService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptionsMonitor<SourceHttpClientOptions> _httpClientOptions;
|
||||
@@ -46,6 +48,7 @@ public sealed class SourceFetchService
|
||||
IHttpClientFactory httpClientFactory,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
MongoContracts.IDocumentStore documentStore,
|
||||
StorageContracts.IStorageDocumentStore storageDocumentStore,
|
||||
ILogger<SourceFetchService> logger,
|
||||
IJitterSource jitterSource,
|
||||
IAdvisoryRawWriteGuard guard,
|
||||
@@ -58,6 +61,7 @@ public sealed class SourceFetchService
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_storageDocumentStore = storageDocumentStore ?? throw new ArgumentNullException(nameof(storageDocumentStore));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_jitterSource = jitterSource ?? throw new ArgumentNullException(nameof(jitterSource));
|
||||
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
|
||||
@@ -69,6 +73,36 @@ public sealed class SourceFetchService
|
||||
_connectorVersion = typeof(SourceFetchService).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||
}
|
||||
|
||||
// Backward-compatible constructor until all callers provide the storage document contract explicitly.
|
||||
public SourceFetchService(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
MongoContracts.IDocumentStore documentStore,
|
||||
ILogger<SourceFetchService> logger,
|
||||
IJitterSource jitterSource,
|
||||
IAdvisoryRawWriteGuard guard,
|
||||
IAdvisoryLinksetMapper linksetMapper,
|
||||
ICryptoHash hash,
|
||||
TimeProvider? timeProvider = null,
|
||||
IOptionsMonitor<SourceHttpClientOptions>? httpClientOptions = null,
|
||||
IOptions<MongoContracts.MongoStorageOptions>? storageOptions = null)
|
||||
: this(
|
||||
httpClientFactory,
|
||||
rawDocumentStorage,
|
||||
documentStore,
|
||||
documentStore as StorageContracts.IStorageDocumentStore
|
||||
?? throw new ArgumentNullException(nameof(documentStore), "Document store must implement IStorageDocumentStore"),
|
||||
logger,
|
||||
jitterSource,
|
||||
guard,
|
||||
linksetMapper,
|
||||
hash,
|
||||
timeProvider,
|
||||
httpClientOptions,
|
||||
storageOptions)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task<SourceFetchResult> FetchAsync(SourceFetchRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
@@ -147,7 +181,7 @@ public sealed class SourceFetchService
|
||||
}
|
||||
}
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(request.SourceName, request.RequestUri.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
var existing = await _storageDocumentStore.FindBySourceAndUriAsync(request.SourceName, request.RequestUri.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
var recordId = existing?.Id ?? Guid.NewGuid();
|
||||
|
||||
var payloadId = await _rawDocumentStorage.UploadAsync(
|
||||
@@ -159,7 +193,7 @@ public sealed class SourceFetchService
|
||||
cancellationToken,
|
||||
recordId).ConfigureAwait(false);
|
||||
|
||||
var record = new MongoContracts.DocumentRecord(
|
||||
var record = new StorageContracts.StorageDocument(
|
||||
recordId,
|
||||
request.SourceName,
|
||||
request.RequestUri.ToString(),
|
||||
@@ -173,9 +207,10 @@ public sealed class SourceFetchService
|
||||
response.Content.Headers.LastModified,
|
||||
payloadId,
|
||||
expiresAt,
|
||||
Payload: contentBytes);
|
||||
Payload: contentBytes,
|
||||
FetchedAt: fetchedAt);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
var upserted = await _storageDocumentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, contentBytes.LongLength, rateLimitRemaining);
|
||||
activity?.SetStatus(ActivityStatusCode.Ok);
|
||||
_logger.LogInformation("Fetched {Source} document {Uri} (sha256={Sha})", request.SourceName, request.RequestUri, contentHash);
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// Postgres-native storage document contract (Mongo-free).
|
||||
/// </summary>
|
||||
public sealed record StorageDocument(
|
||||
Guid Id,
|
||||
string SourceName,
|
||||
string Uri,
|
||||
DateTimeOffset CreatedAt,
|
||||
string Sha256,
|
||||
string Status,
|
||||
string? ContentType,
|
||||
IReadOnlyDictionary<string, string>? Headers,
|
||||
IReadOnlyDictionary<string, string>? Metadata,
|
||||
string? Etag,
|
||||
DateTimeOffset? LastModified,
|
||||
Guid? PayloadId,
|
||||
DateTimeOffset? ExpiresAt,
|
||||
byte[]? Payload,
|
||||
DateTimeOffset? FetchedAt);
|
||||
|
||||
public interface IStorageDocumentStore
|
||||
{
|
||||
Task<StorageDocument?> FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken);
|
||||
Task<StorageDocument?> FindAsync(Guid id, CancellationToken cancellationToken);
|
||||
Task<StorageDocument> UpsertAsync(StorageDocument record, CancellationToken cancellationToken);
|
||||
Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Postgres-native DTO storage contract using JSON payloads.
|
||||
/// </summary>
|
||||
public sealed record StorageDto(
|
||||
Guid Id,
|
||||
Guid DocumentId,
|
||||
string SourceName,
|
||||
string Format,
|
||||
JsonDocument Payload,
|
||||
DateTimeOffset CreatedAt,
|
||||
string SchemaVersion,
|
||||
DateTimeOffset ValidatedAt);
|
||||
|
||||
public interface IStorageDtoStore
|
||||
{
|
||||
Task<StorageDto> UpsertAsync(StorageDto record, CancellationToken cancellationToken);
|
||||
Task<StorageDto?> FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken);
|
||||
Task<IReadOnlyList<StorageDto>> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cursor/state contract for ingestion sources without Mongo/Bson dependencies.
|
||||
/// </summary>
|
||||
public sealed record SourceCursorState(
|
||||
string SourceName,
|
||||
bool Enabled,
|
||||
bool Paused,
|
||||
JsonDocument? Cursor,
|
||||
DateTimeOffset? LastSuccess,
|
||||
DateTimeOffset? LastFailure,
|
||||
int FailCount,
|
||||
DateTimeOffset? BackoffUntil,
|
||||
DateTimeOffset UpdatedAt,
|
||||
string? LastFailureReason);
|
||||
|
||||
public interface ISourceStateStore
|
||||
{
|
||||
Task<SourceCursorState?> TryGetAsync(string sourceName, CancellationToken cancellationToken);
|
||||
Task UpdateCursorAsync(string sourceName, JsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken);
|
||||
Task MarkFailureAsync(string sourceName, DateTimeOffset now, TimeSpan backoff, string reason, CancellationToken cancellationToken);
|
||||
Task UpsertAsync(SourceCursorState record, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.IO;
|
||||
using Contracts = StellaOps.Concelier.Storage.Contracts;
|
||||
using MongoContracts = StellaOps.Concelier.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres;
|
||||
|
||||
internal static class ContractsMappingExtensions
|
||||
{
|
||||
private static readonly JsonWriterSettings RelaxedJsonSettings = new()
|
||||
{
|
||||
OutputMode = JsonOutputMode.RelaxedExtendedJson
|
||||
};
|
||||
|
||||
internal static Contracts.StorageDocument ToStorageDocument(this MongoContracts.DocumentRecord record)
|
||||
{
|
||||
return new Contracts.StorageDocument(
|
||||
record.Id,
|
||||
record.SourceName,
|
||||
record.Uri,
|
||||
record.CreatedAt,
|
||||
record.Sha256,
|
||||
record.Status,
|
||||
record.ContentType,
|
||||
record.Headers,
|
||||
record.Metadata,
|
||||
record.Etag,
|
||||
record.LastModified,
|
||||
record.PayloadId,
|
||||
record.ExpiresAt,
|
||||
record.Payload,
|
||||
record.FetchedAt);
|
||||
}
|
||||
|
||||
internal static MongoContracts.DocumentRecord ToMongoDocumentRecord(this Contracts.StorageDocument record)
|
||||
{
|
||||
return new MongoContracts.DocumentRecord(
|
||||
record.Id,
|
||||
record.SourceName,
|
||||
record.Uri,
|
||||
record.CreatedAt,
|
||||
record.Sha256,
|
||||
record.Status,
|
||||
record.ContentType,
|
||||
record.Headers,
|
||||
record.Metadata,
|
||||
record.Etag,
|
||||
record.LastModified,
|
||||
record.PayloadId,
|
||||
record.ExpiresAt,
|
||||
record.Payload,
|
||||
record.FetchedAt);
|
||||
}
|
||||
|
||||
internal static Contracts.StorageDto ToStorageDto(this MongoContracts.DtoRecord record)
|
||||
{
|
||||
var json = record.Payload.ToJson(RelaxedJsonSettings);
|
||||
var payload = JsonDocument.Parse(json);
|
||||
return new Contracts.StorageDto(
|
||||
record.Id,
|
||||
record.DocumentId,
|
||||
record.SourceName,
|
||||
record.Format,
|
||||
payload,
|
||||
record.CreatedAt,
|
||||
record.SchemaVersion,
|
||||
record.ValidatedAt);
|
||||
}
|
||||
|
||||
internal static MongoContracts.DtoRecord ToMongoDtoRecord(this Contracts.StorageDto record)
|
||||
{
|
||||
var json = record.Payload.RootElement.GetRawText();
|
||||
var bson = BsonDocument.Parse(json);
|
||||
return new MongoContracts.DtoRecord(
|
||||
record.Id,
|
||||
record.DocumentId,
|
||||
record.SourceName,
|
||||
record.Format,
|
||||
bson,
|
||||
record.CreatedAt,
|
||||
record.SchemaVersion,
|
||||
record.ValidatedAt);
|
||||
}
|
||||
|
||||
internal static Contracts.SourceCursorState ToStorageCursorState(this MongoContracts.SourceStateRecord record)
|
||||
{
|
||||
var cursorJson = record.Cursor is null ? null : record.Cursor.ToJson(RelaxedJsonSettings);
|
||||
var cursor = cursorJson is null ? null : JsonDocument.Parse(cursorJson);
|
||||
return new Contracts.SourceCursorState(
|
||||
record.SourceName,
|
||||
record.Enabled,
|
||||
record.Paused,
|
||||
cursor,
|
||||
record.LastSuccess,
|
||||
record.LastFailure,
|
||||
record.FailCount,
|
||||
record.BackoffUntil,
|
||||
record.UpdatedAt,
|
||||
record.LastFailureReason);
|
||||
}
|
||||
|
||||
internal static MongoContracts.SourceStateRecord ToMongoSourceStateRecord(this Contracts.SourceCursorState record)
|
||||
{
|
||||
var bsonCursor = record.Cursor is null ? null : BsonDocument.Parse(record.Cursor.RootElement.GetRawText());
|
||||
return new MongoContracts.SourceStateRecord(
|
||||
record.SourceName,
|
||||
record.Enabled,
|
||||
record.Paused,
|
||||
bsonCursor,
|
||||
record.LastSuccess,
|
||||
record.LastFailure,
|
||||
record.FailCount,
|
||||
record.BackoffUntil,
|
||||
record.UpdatedAt,
|
||||
record.LastFailureReason);
|
||||
}
|
||||
|
||||
internal static BsonDocument ToBsonDocument(this JsonDocument document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
return BsonDocument.Parse(document.RootElement.GetRawText());
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,15 @@
|
||||
using System.Text.Json;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using Contracts = StellaOps.Concelier.Storage.Contracts;
|
||||
using StellaOps.Concelier.Storage.Postgres.Models;
|
||||
using StellaOps.Concelier.Storage.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres;
|
||||
|
||||
/// <summary>
|
||||
/// Postgres-backed implementation that satisfies the legacy IDocumentStore contract.
|
||||
/// Postgres-backed implementation that satisfies the legacy IDocumentStore contract and the new Postgres-native storage contract.
|
||||
/// </summary>
|
||||
public sealed class PostgresDocumentStore : IDocumentStore
|
||||
public sealed class PostgresDocumentStore : IDocumentStore, Contracts.IStorageDocumentStore
|
||||
{
|
||||
private readonly IDocumentRepository _repository;
|
||||
private readonly ISourceRepository _sourceRepository;
|
||||
@@ -64,6 +65,18 @@ public sealed class PostgresDocumentStore : IDocumentStore
|
||||
await _repository.UpdateStatusAsync(id, status, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
async Task<Contracts.StorageDocument?> Contracts.IStorageDocumentStore.FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken)
|
||||
=> (await FindBySourceAndUriAsync(sourceName, uri, cancellationToken).ConfigureAwait(false))?.ToStorageDocument();
|
||||
|
||||
async Task<Contracts.StorageDocument?> Contracts.IStorageDocumentStore.FindAsync(Guid id, CancellationToken cancellationToken)
|
||||
=> (await FindAsync(id, cancellationToken).ConfigureAwait(false))?.ToStorageDocument();
|
||||
|
||||
async Task<Contracts.StorageDocument> Contracts.IStorageDocumentStore.UpsertAsync(Contracts.StorageDocument record, CancellationToken cancellationToken)
|
||||
=> (await UpsertAsync(record.ToMongoDocumentRecord(), cancellationToken).ConfigureAwait(false)).ToStorageDocument();
|
||||
|
||||
Task Contracts.IStorageDocumentStore.UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken)
|
||||
=> UpdateStatusAsync(id, status, cancellationToken);
|
||||
|
||||
private DocumentRecord Map(DocumentRecordEntity row)
|
||||
{
|
||||
return new DocumentRecord(
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using Dapper;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using Contracts = StellaOps.Concelier.Storage.Contracts;
|
||||
using StellaOps.Concelier.Storage.Postgres;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres.Repositories;
|
||||
|
||||
internal sealed class PostgresDtoStore : IDtoStore
|
||||
internal sealed class PostgresDtoStore : IDtoStore, Contracts.IStorageDtoStore
|
||||
{
|
||||
private readonly ConcelierDataSource _dataSource;
|
||||
private readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.General)
|
||||
@@ -92,6 +95,17 @@ internal sealed class PostgresDtoStore : IDtoStore
|
||||
row.ValidatedAt);
|
||||
}
|
||||
|
||||
async Task<Contracts.StorageDto> Contracts.IStorageDtoStore.UpsertAsync(Contracts.StorageDto record, CancellationToken cancellationToken)
|
||||
=> (await UpsertAsync(record.ToMongoDtoRecord(), cancellationToken).ConfigureAwait(false)).ToStorageDto();
|
||||
|
||||
async Task<Contracts.StorageDto?> Contracts.IStorageDtoStore.FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken)
|
||||
=> (await FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false))?.ToStorageDto();
|
||||
|
||||
async Task<IReadOnlyList<Contracts.StorageDto>> Contracts.IStorageDtoStore.GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken)
|
||||
=> (await GetBySourceAsync(sourceName, limit, cancellationToken).ConfigureAwait(false))
|
||||
.Select(dto => dto.ToStorageDto())
|
||||
.ToArray();
|
||||
|
||||
private sealed record DtoRow(
|
||||
Guid Id,
|
||||
Guid DocumentId,
|
||||
|
||||
@@ -4,14 +4,15 @@ using System.Collections.Generic;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Storage.Postgres.Models;
|
||||
using StellaOps.Concelier.Storage.Postgres.Repositories;
|
||||
using Contracts = StellaOps.Concelier.Storage.Contracts;
|
||||
using MongoContracts = StellaOps.Concelier.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres;
|
||||
|
||||
/// <summary>
|
||||
/// Adapter that satisfies the legacy source state contract using PostgreSQL storage.
|
||||
/// Adapter that satisfies the legacy source state contract using PostgreSQL storage and provides a Postgres-native cursor contract.
|
||||
/// </summary>
|
||||
public sealed class PostgresSourceStateAdapter : MongoContracts.ISourceStateRepository
|
||||
public sealed class PostgresSourceStateAdapter : MongoContracts.ISourceStateRepository, Contracts.ISourceStateStore
|
||||
{
|
||||
private readonly ISourceRepository _sourceRepository;
|
||||
private readonly Repositories.ISourceStateRepository _stateRepository;
|
||||
@@ -134,6 +135,18 @@ public sealed class PostgresSourceStateAdapter : MongoContracts.ISourceStateRepo
|
||||
_ = await _stateRepository.UpsertAsync(entity, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
async Task<Contracts.SourceCursorState?> Contracts.ISourceStateStore.TryGetAsync(string sourceName, CancellationToken cancellationToken)
|
||||
=> (await TryGetAsync(sourceName, cancellationToken).ConfigureAwait(false))?.ToStorageCursorState();
|
||||
|
||||
Task Contracts.ISourceStateStore.UpdateCursorAsync(string sourceName, JsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken)
|
||||
=> UpdateCursorAsync(sourceName, cursor.ToBsonDocument(), completedAt, cancellationToken);
|
||||
|
||||
Task Contracts.ISourceStateStore.MarkFailureAsync(string sourceName, DateTimeOffset now, TimeSpan backoff, string reason, CancellationToken cancellationToken)
|
||||
=> MarkFailureAsync(sourceName, now, backoff, reason, cancellationToken);
|
||||
|
||||
Task Contracts.ISourceStateStore.UpsertAsync(Contracts.SourceCursorState record, CancellationToken cancellationToken)
|
||||
=> UpsertAsync(record.ToMongoSourceStateRecord(), cancellationToken);
|
||||
|
||||
private async Task<SourceEntity> EnsureSourceAsync(string sourceName, CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await _sourceRepository.GetByKeyAsync(sourceName, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
- Mission (current sprint): air-gap parity for evidence chunks, trust connector wiring, and attestation verification aligned to Evidence Locker contract.
|
||||
|
||||
## Roles
|
||||
- **Backend engineer (ASP.NET Core / Mongo):** chunk ingestion/export, attestation verifier, trust connector.
|
||||
- **Backend engineer (ASP.NET Core / Postgres):** chunk ingestion/export, attestation verifier, trust connector.
|
||||
- **Air-Gap/Platform engineer:** sealed-mode switches, offline bundles, deterministic cache/path handling.
|
||||
- **QA automation:** WebApplicationFactory + Mongo2Go tests for chunk APIs, attestations, and trust connector; deterministic ordering/hashes.
|
||||
- **QA automation:** WebApplicationFactory + Postgres or in-memory fixtures for chunk APIs, attestations, and trust connector; deterministic ordering/hashes.
|
||||
- **Docs/Schema steward:** keep chunk API, attestation plan, and trust connector docs in sync with behavior; update schemas and samples.
|
||||
|
||||
## Required Reading (treat as read before DOING)
|
||||
@@ -29,7 +29,7 @@
|
||||
- Cross-module edits: require sprint note; otherwise, stay within Excititor working dir.
|
||||
|
||||
## Testing Rules
|
||||
- Use Mongo2Go/in-memory fixtures; avoid network.
|
||||
- Prefer Postgres integration or in-memory fixtures; avoid network.
|
||||
- API tests in `StellaOps.Excititor.WebService.Tests`; worker/connectors in `StellaOps.Excititor.Worker.Tests`; shared fixtures in `__Tests`.
|
||||
- Tests must assert determinism (ordering/hashes), tenant enforcement, and sealed-mode behavior.
|
||||
|
||||
@@ -39,6 +39,6 @@
|
||||
- If a decision is needed, mark the task BLOCKED and record the decision ask—do not pause work.
|
||||
|
||||
## Tooling/Env Notes
|
||||
- .NET 10 with preview features enabled; Mongo driver ≥ 3.x.
|
||||
- .NET 10 with preview features enabled; Postgres or in-memory storage only (Mongo/BSON removed).
|
||||
- Signing/verifier hooks rely on Evidence Locker contract fixtures under `docs/modules/evidence-locker/`.
|
||||
- Sealed-mode tests should run with `EXCITITOR_SEALED=1` (env var) to enforce offline code paths.
|
||||
|
||||
@@ -27,14 +27,15 @@ Expose Excititor APIs (console VEX views, graph/Vuln Explorer feeds, observation
|
||||
5. Observability: structured logs, counters, optional OTEL traces behind configuration flags.
|
||||
|
||||
## Testing
|
||||
- Prefer deterministic API/integration tests under `__Tests` with seeded Mongo fixtures.
|
||||
- Prefer deterministic API/integration tests under `__Tests` with seeded Postgres fixtures or in-memory stores.
|
||||
- Verify RBAC/tenant isolation, idempotent ingestion, and stable ordering of VEX aggregates.
|
||||
- Use ISO-8601 UTC timestamps and stable sorting in responses; assert on content hashes where applicable.
|
||||
|
||||
## Determinism & Data
|
||||
- MongoDB is the canonical store; never apply consensus transformations before persistence.
|
||||
- Postgres append-only storage is canonical; never apply consensus transformations before persistence.
|
||||
- Ensure paged/list endpoints use explicit sort keys (e.g., vendor, upstreamId, version, createdUtc).
|
||||
- Avoid nondeterministic clocks/randomness; inject clocks and GUID providers for tests.
|
||||
- Evidence/attestation endpoints are temporarily disabled; re-enable only when Postgres-backed stores land (Mongo/BSON removed).
|
||||
|
||||
## Boundaries
|
||||
- Do not modify Policy Engine or Cartographer schemas from here; consume published contracts only.
|
||||
|
||||
@@ -1,40 +1,23 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
/// <summary>
|
||||
/// Attestation API endpoints (WEB-OBS-54-001).
|
||||
/// Exposes /attestations/vex/* endpoints returning DSSE verification state,
|
||||
/// builder identity, and chain-of-custody links.
|
||||
/// Attestation API endpoints (temporarily disabled while Mongo is removed and Postgres storage is adopted).
|
||||
/// </summary>
|
||||
public static class AttestationEndpoints
|
||||
{
|
||||
public static void MapAttestationEndpoints(this WebApplication app)
|
||||
{
|
||||
// GET /attestations/vex/list - List attestations
|
||||
app.MapGet("/attestations/vex/list", async (
|
||||
// GET /attestations/vex/list
|
||||
app.MapGet("/attestations/vex/list", (
|
||||
HttpContext context,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IMongoDatabase database,
|
||||
TimeProvider timeProvider,
|
||||
[FromQuery] int? limit,
|
||||
[FromQuery] string? cursor,
|
||||
[FromQuery] string? vulnerabilityId,
|
||||
[FromQuery] string? productKey,
|
||||
CancellationToken cancellationToken) =>
|
||||
IOptions<VexStorageOptions> storageOptions) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -42,70 +25,22 @@ public static class AttestationEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200);
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Attestations);
|
||||
var builder = Builders<BsonDocument>.Filter;
|
||||
var filters = new List<FilterDefinition<BsonDocument>>();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(vulnerabilityId))
|
||||
{
|
||||
filters.Add(builder.Eq("VulnerabilityId", vulnerabilityId.Trim().ToUpperInvariant()));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
filters.Add(builder.Eq("ProductKey", productKey.Trim().ToLowerInvariant()));
|
||||
}
|
||||
|
||||
// Parse cursor if provided
|
||||
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
|
||||
{
|
||||
var ltTime = builder.Lt("IssuedAt", cursorTime);
|
||||
var eqTimeLtId = builder.And(
|
||||
builder.Eq("IssuedAt", cursorTime),
|
||||
builder.Lt("_id", cursorId));
|
||||
filters.Add(builder.Or(ltTime, eqTimeLtId));
|
||||
}
|
||||
|
||||
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
|
||||
var sort = Builders<BsonDocument>.Sort.Descending("IssuedAt").Descending("_id");
|
||||
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.Sort(sort)
|
||||
.Limit(take)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var items = documents.Select(doc => ToListItem(doc, tenant, timeProvider)).ToList();
|
||||
|
||||
string? nextCursor = null;
|
||||
var hasMore = documents.Count == take;
|
||||
if (hasMore && documents.Count > 0)
|
||||
{
|
||||
var last = documents[^1];
|
||||
var lastTime = last.GetValue("IssuedAt", BsonNull.Value).ToUniversalTime();
|
||||
var lastId = last.GetValue("_id", BsonNull.Value).AsString;
|
||||
nextCursor = EncodeCursor(lastTime, lastId);
|
||||
}
|
||||
|
||||
var response = new VexAttestationListResponse(items, nextCursor, hasMore, items.Count);
|
||||
return Results.Ok(response);
|
||||
return Results.Problem(
|
||||
detail: "Attestation listing is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
}).WithName("ListVexAttestations");
|
||||
|
||||
// GET /attestations/vex/{attestationId} - Get attestation details
|
||||
app.MapGet("/attestations/vex/{attestationId}", async (
|
||||
// GET /attestations/vex/{attestationId}
|
||||
app.MapGet("/attestations/vex/{attestationId}", (
|
||||
HttpContext context,
|
||||
string attestationId,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IVexAttestationLinkStore attestationStore,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
IOptions<VexStorageOptions> storageOptions) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -113,235 +48,23 @@ public static class AttestationEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(attestationId))
|
||||
{
|
||||
return Results.BadRequest(new { error = new { code = "ERR_ATTESTATION_ID", message = "attestationId is required" } });
|
||||
return Results.Problem(
|
||||
detail: "attestationId is required.",
|
||||
statusCode: StatusCodes.Status400BadRequest,
|
||||
title: "Validation error");
|
||||
}
|
||||
|
||||
var attestation = await attestationStore.FindAsync(attestationId.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
if (attestation is null)
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = $"Attestation '{attestationId}' not found" } });
|
||||
}
|
||||
|
||||
// Build subject from observation context
|
||||
var subjectDigest = attestation.Metadata.TryGetValue("digest", out var dig) ? dig : attestation.ObservationId;
|
||||
var subject = new VexAttestationSubject(
|
||||
Digest: subjectDigest,
|
||||
DigestAlgorithm: "sha256",
|
||||
Name: $"{attestation.VulnerabilityId}/{attestation.ProductKey}",
|
||||
Uri: null);
|
||||
|
||||
var builder = new VexAttestationBuilderIdentity(
|
||||
Id: attestation.SupplierId,
|
||||
Version: null,
|
||||
BuilderId: attestation.SupplierId,
|
||||
InvocationId: attestation.ObservationId);
|
||||
|
||||
// Get verification state from metadata
|
||||
var isValid = attestation.Metadata.TryGetValue("verified", out var verified) && verified == "true";
|
||||
DateTimeOffset? verifiedAt = null;
|
||||
if (attestation.Metadata.TryGetValue("verifiedAt", out var verifiedAtStr) &&
|
||||
DateTimeOffset.TryParse(verifiedAtStr, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsedVerifiedAt))
|
||||
{
|
||||
verifiedAt = parsedVerifiedAt;
|
||||
}
|
||||
|
||||
var verification = new VexAttestationVerificationState(
|
||||
Valid: isValid,
|
||||
VerifiedAt: verifiedAt,
|
||||
SignatureType: attestation.Metadata.GetValueOrDefault("signatureType", "dsse"),
|
||||
KeyId: attestation.Metadata.GetValueOrDefault("keyId"),
|
||||
Issuer: attestation.Metadata.GetValueOrDefault("issuer"),
|
||||
EnvelopeDigest: attestation.Metadata.GetValueOrDefault("envelopeDigest"),
|
||||
Diagnostics: attestation.Metadata);
|
||||
|
||||
var custodyLinks = new List<VexAttestationCustodyLink>
|
||||
{
|
||||
new(
|
||||
Step: 1,
|
||||
Actor: attestation.SupplierId,
|
||||
Action: "created",
|
||||
Timestamp: attestation.IssuedAt,
|
||||
Reference: attestation.AttestationId)
|
||||
};
|
||||
|
||||
// Add linkset link
|
||||
custodyLinks.Add(new VexAttestationCustodyLink(
|
||||
Step: 2,
|
||||
Actor: "excititor",
|
||||
Action: "linked_to_observation",
|
||||
Timestamp: attestation.IssuedAt,
|
||||
Reference: attestation.LinksetId));
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["observationId"] = attestation.ObservationId,
|
||||
["linksetId"] = attestation.LinksetId,
|
||||
["vulnerabilityId"] = attestation.VulnerabilityId,
|
||||
["productKey"] = attestation.ProductKey
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(attestation.JustificationSummary))
|
||||
{
|
||||
metadata["justificationSummary"] = attestation.JustificationSummary;
|
||||
}
|
||||
|
||||
var response = new VexAttestationDetailResponse(
|
||||
AttestationId: attestation.AttestationId,
|
||||
Tenant: tenant,
|
||||
CreatedAt: attestation.IssuedAt,
|
||||
PredicateType: attestation.Metadata.GetValueOrDefault("predicateType", "https://in-toto.io/attestation/v1"),
|
||||
Subject: subject,
|
||||
Builder: builder,
|
||||
Verification: verification,
|
||||
ChainOfCustody: custodyLinks,
|
||||
Metadata: metadata);
|
||||
|
||||
return Results.Ok(response);
|
||||
return Results.Problem(
|
||||
detail: "Attestation retrieval is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
}).WithName("GetVexAttestation");
|
||||
|
||||
// GET /attestations/vex/lookup - Lookup attestations by linkset or observation
|
||||
app.MapGet("/attestations/vex/lookup", async (
|
||||
HttpContext context,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IMongoDatabase database,
|
||||
TimeProvider timeProvider,
|
||||
[FromQuery] string? linksetId,
|
||||
[FromQuery] string? observationId,
|
||||
[FromQuery] int? limit,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(linksetId) && string.IsNullOrWhiteSpace(observationId))
|
||||
{
|
||||
return Results.BadRequest(new { error = new { code = "ERR_PARAMS", message = "Either linksetId or observationId is required" } });
|
||||
}
|
||||
|
||||
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 100);
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Attestations);
|
||||
var builder = Builders<BsonDocument>.Filter;
|
||||
|
||||
FilterDefinition<BsonDocument> filter;
|
||||
if (!string.IsNullOrWhiteSpace(linksetId))
|
||||
{
|
||||
filter = builder.Eq("LinksetId", linksetId.Trim());
|
||||
}
|
||||
else
|
||||
{
|
||||
filter = builder.Eq("ObservationId", observationId!.Trim());
|
||||
}
|
||||
|
||||
var sort = Builders<BsonDocument>.Sort.Descending("IssuedAt");
|
||||
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.Sort(sort)
|
||||
.Limit(take)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var items = documents.Select(doc => ToListItem(doc, tenant, timeProvider)).ToList();
|
||||
|
||||
var response = new VexAttestationLookupResponse(
|
||||
SubjectDigest: linksetId ?? observationId ?? string.Empty,
|
||||
Attestations: items,
|
||||
QueriedAt: timeProvider.GetUtcNow());
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("LookupVexAttestations");
|
||||
}
|
||||
|
||||
private static VexAttestationListItem ToListItem(BsonDocument doc, string tenant, TimeProvider timeProvider)
|
||||
{
|
||||
return new VexAttestationListItem(
|
||||
AttestationId: doc.GetValue("_id", BsonNull.Value).AsString ?? string.Empty,
|
||||
Tenant: tenant,
|
||||
CreatedAt: doc.GetValue("IssuedAt", BsonNull.Value).IsBsonDateTime
|
||||
? new DateTimeOffset(doc["IssuedAt"].ToUniversalTime(), TimeSpan.Zero)
|
||||
: timeProvider.GetUtcNow(),
|
||||
PredicateType: "https://in-toto.io/attestation/v1",
|
||||
SubjectDigest: doc.GetValue("ObservationId", BsonNull.Value).AsString ?? string.Empty,
|
||||
Valid: doc.Contains("Metadata") && !doc["Metadata"].IsBsonNull &&
|
||||
doc["Metadata"].AsBsonDocument.Contains("verified") &&
|
||||
doc["Metadata"]["verified"].AsString == "true",
|
||||
BuilderId: doc.GetValue("SupplierId", BsonNull.Value).AsString);
|
||||
}
|
||||
|
||||
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, out string tenant, out IResult? problem)
|
||||
{
|
||||
tenant = options.DefaultTenant;
|
||||
problem = null;
|
||||
|
||||
if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerValues) && headerValues.Count > 0)
|
||||
{
|
||||
var requestedTenant = headerValues[0]?.Trim();
|
||||
if (string.IsNullOrEmpty(requestedTenant))
|
||||
{
|
||||
problem = Results.BadRequest(new { error = new { code = "ERR_TENANT", message = "X-Stella-Tenant header must not be empty" } });
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!string.Equals(requestedTenant, options.DefaultTenant, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
problem = Results.Json(
|
||||
new { error = new { code = "ERR_TENANT_FORBIDDEN", message = $"Tenant '{requestedTenant}' is not allowed" } },
|
||||
statusCode: StatusCodes.Status403Forbidden);
|
||||
return false;
|
||||
}
|
||||
|
||||
tenant = requestedTenant;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryDecodeCursor(string cursor, out DateTime timestamp, out string id)
|
||||
{
|
||||
timestamp = default;
|
||||
id = string.Empty;
|
||||
try
|
||||
{
|
||||
var payload = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(cursor));
|
||||
var parts = payload.Split('|');
|
||||
if (parts.Length != 2)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParse(parts[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
timestamp = parsed.UtcDateTime;
|
||||
id = parts[1];
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string EncodeCursor(DateTime timestamp, string id)
|
||||
{
|
||||
var payload = FormattableString.Invariant($"{timestamp:O}|{id}");
|
||||
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(payload));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,48 +1,24 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Canonicalization;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using StellaOps.Excititor.WebService.Telemetry;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Endpoints;
|
||||
|
||||
/// <summary>
|
||||
/// Evidence API endpoints (WEB-OBS-53-001).
|
||||
/// Exposes /evidence/vex/* endpoints that fetch locker bundles, enforce scopes,
|
||||
/// and surface verification metadata without synthesizing verdicts.
|
||||
/// Evidence API endpoints (temporarily disabled while Mongo/BSON storage is removed).
|
||||
/// </summary>
|
||||
public static class EvidenceEndpoints
|
||||
{
|
||||
public static void MapEvidenceEndpoints(this WebApplication app)
|
||||
{
|
||||
// GET /evidence/vex/list - List evidence exports
|
||||
app.MapGet("/evidence/vex/list", async (
|
||||
// GET /evidence/vex/list
|
||||
app.MapGet("/evidence/vex/list", (
|
||||
HttpContext context,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IMongoDatabase database,
|
||||
TimeProvider timeProvider,
|
||||
[FromQuery] int? limit,
|
||||
[FromQuery] string? cursor,
|
||||
[FromQuery] string? format,
|
||||
CancellationToken cancellationToken) =>
|
||||
ChunkTelemetry chunkTelemetry) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -50,74 +26,23 @@ public static class EvidenceEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200);
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Exports);
|
||||
var builder = Builders<BsonDocument>.Filter;
|
||||
var filters = new List<FilterDefinition<BsonDocument>>();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(format))
|
||||
{
|
||||
filters.Add(builder.Eq("Format", format.Trim().ToLowerInvariant()));
|
||||
}
|
||||
|
||||
// Parse cursor if provided (base64-encoded timestamp|id)
|
||||
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
|
||||
{
|
||||
var ltTime = builder.Lt("CreatedAt", cursorTime);
|
||||
var eqTimeLtId = builder.And(
|
||||
builder.Eq("CreatedAt", cursorTime),
|
||||
builder.Lt("_id", cursorId));
|
||||
filters.Add(builder.Or(ltTime, eqTimeLtId));
|
||||
}
|
||||
|
||||
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
|
||||
var sort = Builders<BsonDocument>.Sort.Descending("CreatedAt").Descending("_id");
|
||||
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.Sort(sort)
|
||||
.Limit(take)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var items = documents.Select(doc => new VexEvidenceListItem(
|
||||
BundleId: doc.GetValue("ExportId", BsonNull.Value).AsString ?? doc.GetValue("_id", BsonNull.Value).AsString,
|
||||
Tenant: tenant,
|
||||
CreatedAt: doc.GetValue("CreatedAt", BsonNull.Value).IsBsonDateTime
|
||||
? new DateTimeOffset(doc["CreatedAt"].ToUniversalTime(), TimeSpan.Zero)
|
||||
: timeProvider.GetUtcNow(),
|
||||
ContentHash: doc.GetValue("ArtifactDigest", BsonNull.Value).AsString ?? string.Empty,
|
||||
Format: doc.GetValue("Format", BsonNull.Value).AsString ?? "json",
|
||||
ItemCount: doc.GetValue("ClaimCount", BsonNull.Value).IsInt32 ? doc["ClaimCount"].AsInt32 : 0,
|
||||
Verified: doc.Contains("Attestation") && !doc["Attestation"].IsBsonNull)).ToList();
|
||||
|
||||
string? nextCursor = null;
|
||||
var hasMore = documents.Count == take;
|
||||
if (hasMore && documents.Count > 0)
|
||||
{
|
||||
var last = documents[^1];
|
||||
var lastTime = last.GetValue("CreatedAt", BsonNull.Value).ToUniversalTime();
|
||||
var lastId = last.GetValue("_id", BsonNull.Value).AsString;
|
||||
nextCursor = EncodeCursor(lastTime, lastId);
|
||||
}
|
||||
|
||||
var response = new VexEvidenceListResponse(items, nextCursor, hasMore, items.Count);
|
||||
return Results.Ok(response);
|
||||
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
|
||||
return Results.Problem(
|
||||
detail: "Evidence exports are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
}).WithName("ListVexEvidence");
|
||||
|
||||
// GET /evidence/vex/bundle/{bundleId} - Get evidence bundle details
|
||||
app.MapGet("/evidence/vex/bundle/{bundleId}", async (
|
||||
// GET /evidence/vex/{bundleId}
|
||||
app.MapGet("/evidence/vex/{bundleId}", (
|
||||
HttpContext context,
|
||||
string bundleId,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IMongoDatabase database,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
IOptions<VexStorageOptions> storageOptions) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -125,79 +50,30 @@ public static class EvidenceEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(bundleId))
|
||||
{
|
||||
return Results.BadRequest(new { error = new { code = "ERR_BUNDLE_ID", message = "bundleId is required" } });
|
||||
return Results.Problem(
|
||||
detail: "bundleId is required.",
|
||||
statusCode: StatusCodes.Status400BadRequest,
|
||||
title: "Validation error");
|
||||
}
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Exports);
|
||||
var filter = Builders<BsonDocument>.Filter.Or(
|
||||
Builders<BsonDocument>.Filter.Eq("_id", bundleId.Trim()),
|
||||
Builders<BsonDocument>.Filter.Eq("ExportId", bundleId.Trim()));
|
||||
|
||||
var doc = await collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (doc is null)
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = $"Evidence bundle '{bundleId}' not found" } });
|
||||
}
|
||||
|
||||
VexEvidenceVerificationMetadata? verification = null;
|
||||
if (doc.Contains("Attestation") && !doc["Attestation"].IsBsonNull)
|
||||
{
|
||||
var att = doc["Attestation"].AsBsonDocument;
|
||||
verification = new VexEvidenceVerificationMetadata(
|
||||
Verified: true,
|
||||
VerifiedAt: att.Contains("SignedAt") && att["SignedAt"].IsBsonDateTime
|
||||
? new DateTimeOffset(att["SignedAt"].ToUniversalTime(), TimeSpan.Zero)
|
||||
: null,
|
||||
SignatureType: "dsse",
|
||||
KeyId: att.GetValue("KeyId", BsonNull.Value).AsString,
|
||||
Issuer: att.GetValue("Issuer", BsonNull.Value).AsString,
|
||||
TransparencyRef: att.Contains("Rekor") && !att["Rekor"].IsBsonNull
|
||||
? att["Rekor"].AsBsonDocument.GetValue("Location", BsonNull.Value).AsString
|
||||
: null);
|
||||
}
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
if (doc.Contains("SourceProviders") && doc["SourceProviders"].IsBsonArray)
|
||||
{
|
||||
metadata["sourceProviders"] = string.Join(",", doc["SourceProviders"].AsBsonArray.Select(v => v.AsString));
|
||||
}
|
||||
if (doc.Contains("PolicyRevisionId") && !doc["PolicyRevisionId"].IsBsonNull)
|
||||
{
|
||||
metadata["policyRevisionId"] = doc["PolicyRevisionId"].AsString;
|
||||
}
|
||||
|
||||
var response = new VexEvidenceBundleResponse(
|
||||
BundleId: doc.GetValue("ExportId", BsonNull.Value).AsString ?? bundleId.Trim(),
|
||||
Tenant: tenant,
|
||||
CreatedAt: doc.GetValue("CreatedAt", BsonNull.Value).IsBsonDateTime
|
||||
? new DateTimeOffset(doc["CreatedAt"].ToUniversalTime(), TimeSpan.Zero)
|
||||
: timeProvider.GetUtcNow(),
|
||||
ContentHash: doc.GetValue("ArtifactDigest", BsonNull.Value).AsString ?? string.Empty,
|
||||
Format: doc.GetValue("Format", BsonNull.Value).AsString ?? "json",
|
||||
ItemCount: doc.GetValue("ClaimCount", BsonNull.Value).IsInt32 ? doc["ClaimCount"].AsInt32 : 0,
|
||||
Verification: verification,
|
||||
Metadata: metadata);
|
||||
|
||||
return Results.Ok(response);
|
||||
return Results.Problem(
|
||||
detail: "Evidence bundles are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
}).WithName("GetVexEvidenceBundle");
|
||||
|
||||
// GET /evidence/vex/lookup - Lookup evidence for vuln/product pair
|
||||
app.MapGet("/evidence/vex/lookup", async (
|
||||
// GET /v1/vex/evidence/chunks
|
||||
app.MapGet("/v1/vex/evidence/chunks", (
|
||||
HttpContext context,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IVexObservationProjectionService projectionService,
|
||||
TimeProvider timeProvider,
|
||||
[FromQuery] string vulnerabilityId,
|
||||
[FromQuery] string productKey,
|
||||
[FromQuery] int? limit,
|
||||
CancellationToken cancellationToken) =>
|
||||
ChunkTelemetry chunkTelemetry) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
@@ -205,572 +81,16 @@ public static class EvidenceEndpoints
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
return Results.BadRequest(new { error = new { code = "ERR_PARAMS", message = "vulnerabilityId and productKey are required" } });
|
||||
}
|
||||
|
||||
var take = Math.Clamp(limit.GetValueOrDefault(100), 1, 500);
|
||||
var request = new VexObservationProjectionRequest(
|
||||
tenant,
|
||||
vulnerabilityId.Trim(),
|
||||
productKey.Trim(),
|
||||
ImmutableHashSet<string>.Empty,
|
||||
ImmutableHashSet<VexClaimStatus>.Empty,
|
||||
null,
|
||||
take);
|
||||
|
||||
var result = await projectionService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var items = result.Statements.Select(s => new VexEvidenceItem(
|
||||
ObservationId: s.ObservationId,
|
||||
ProviderId: s.ProviderId,
|
||||
Status: s.Status.ToString().ToLowerInvariant(),
|
||||
Justification: s.Justification?.ToString().ToLowerInvariant(),
|
||||
FirstSeen: s.FirstSeen,
|
||||
LastSeen: s.LastSeen,
|
||||
DocumentDigest: s.Document.Digest,
|
||||
Verification: s.Signature is null ? null : new VexEvidenceVerificationMetadata(
|
||||
Verified: s.Signature.VerifiedAt.HasValue,
|
||||
VerifiedAt: s.Signature.VerifiedAt,
|
||||
SignatureType: s.Signature.Type,
|
||||
KeyId: s.Signature.KeyId,
|
||||
Issuer: s.Signature.Issuer,
|
||||
TransparencyRef: null))).ToList();
|
||||
|
||||
var response = new VexEvidenceLookupResponse(
|
||||
VulnerabilityId: vulnerabilityId.Trim(),
|
||||
ProductKey: productKey.Trim(),
|
||||
EvidenceItems: items,
|
||||
QueriedAt: timeProvider.GetUtcNow());
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("LookupVexEvidence");
|
||||
|
||||
// GET /vuln/evidence/vex/{advisory_key} - Get evidence by advisory key (EXCITITOR-VULN-29-002)
|
||||
app.MapGet("/vuln/evidence/vex/{advisory_key}", async (
|
||||
HttpContext context,
|
||||
string advisory_key,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
[FromServices] IMongoDatabase database,
|
||||
TimeProvider timeProvider,
|
||||
[FromQuery] int? limit,
|
||||
[FromQuery] string? cursor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(advisory_key))
|
||||
{
|
||||
NormalizationTelemetry.RecordAdvisoryKeyCanonicalizeError(tenant, "empty_key");
|
||||
return Results.BadRequest(new { error = new { code = "ERR_ADVISORY_KEY", message = "advisory_key is required" } });
|
||||
}
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
// Canonicalize the advisory key using VexAdvisoryKeyCanonicalizer
|
||||
var canonicalizer = new VexAdvisoryKeyCanonicalizer();
|
||||
VexCanonicalAdvisoryKey canonicalKey;
|
||||
try
|
||||
{
|
||||
canonicalKey = canonicalizer.Canonicalize(advisory_key.Trim());
|
||||
NormalizationTelemetry.RecordAdvisoryKeyCanonicalization(tenant, canonicalKey);
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
NormalizationTelemetry.RecordAdvisoryKeyCanonicalizeError(tenant, "invalid_format", advisory_key);
|
||||
return Results.BadRequest(new { error = new { code = "ERR_INVALID_ADVISORY_KEY", message = ex.Message } });
|
||||
}
|
||||
|
||||
var take = Math.Clamp(limit.GetValueOrDefault(100), 1, 500);
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Statements);
|
||||
var builder = Builders<BsonDocument>.Filter;
|
||||
|
||||
// Build filter to match by vulnerability ID (case-insensitive)
|
||||
// Try original key, canonical key, and all aliases
|
||||
var vulnerabilityFilters = new List<FilterDefinition<BsonDocument>>
|
||||
{
|
||||
builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(advisory_key.Trim())}$", "i"))
|
||||
};
|
||||
|
||||
// Add canonical key if different
|
||||
if (!string.Equals(canonicalKey.AdvisoryKey, advisory_key.Trim(), StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
vulnerabilityFilters.Add(builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(canonicalKey.AdvisoryKey)}$", "i")));
|
||||
}
|
||||
|
||||
// Add original ID if available
|
||||
if (canonicalKey.OriginalId is { } originalId &&
|
||||
!string.Equals(originalId, advisory_key.Trim(), StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
vulnerabilityFilters.Add(builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(originalId)}$", "i")));
|
||||
}
|
||||
|
||||
var filter = builder.Or(vulnerabilityFilters);
|
||||
|
||||
// Apply cursor-based pagination if provided
|
||||
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
|
||||
{
|
||||
var ltTime = builder.Lt("InsertedAt", cursorTime);
|
||||
var eqTimeLtId = builder.And(
|
||||
builder.Eq("InsertedAt", cursorTime),
|
||||
builder.Lt("_id", ObjectId.Parse(cursorId)));
|
||||
filter = builder.And(filter, builder.Or(ltTime, eqTimeLtId));
|
||||
}
|
||||
|
||||
var sort = Builders<BsonDocument>.Sort.Descending("InsertedAt").Descending("_id");
|
||||
|
||||
var documents = await collection
|
||||
.Find(filter)
|
||||
.Sort(sort)
|
||||
.Limit(take)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var statements = new List<VexAdvisoryStatementResponse>();
|
||||
|
||||
foreach (var doc in documents)
|
||||
{
|
||||
var provenance = new VexAdvisoryProvenanceResponse(
|
||||
DocumentDigest: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
|
||||
? doc["Document"].AsBsonDocument.GetValue("Digest", BsonNull.Value).AsString ?? string.Empty
|
||||
: string.Empty,
|
||||
DocumentFormat: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
|
||||
? doc["Document"].AsBsonDocument.GetValue("Format", BsonNull.Value).AsString ?? "unknown"
|
||||
: "unknown",
|
||||
SourceUri: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
|
||||
? doc["Document"].AsBsonDocument.GetValue("SourceUri", BsonNull.Value).AsString ?? string.Empty
|
||||
: string.Empty,
|
||||
Revision: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
|
||||
? doc["Document"].AsBsonDocument.GetValue("Revision", BsonNull.Value).AsString
|
||||
: null,
|
||||
InsertedAt: doc.GetValue("InsertedAt", BsonNull.Value).IsBsonDateTime
|
||||
? new DateTimeOffset(doc["InsertedAt"].ToUniversalTime(), TimeSpan.Zero)
|
||||
: now);
|
||||
|
||||
VexAdvisoryAttestationResponse? attestation = null;
|
||||
if (doc.GetValue("Document", BsonNull.Value).IsBsonDocument)
|
||||
{
|
||||
var docSection = doc["Document"].AsBsonDocument;
|
||||
if (docSection.Contains("Signature") && !docSection["Signature"].IsBsonNull)
|
||||
{
|
||||
var sig = docSection["Signature"].AsBsonDocument;
|
||||
var sigType = sig.GetValue("Type", BsonNull.Value).AsString;
|
||||
if (!string.IsNullOrWhiteSpace(sigType))
|
||||
{
|
||||
attestation = new VexAdvisoryAttestationResponse(
|
||||
SignatureType: sigType,
|
||||
Issuer: sig.GetValue("Issuer", BsonNull.Value).AsString,
|
||||
Subject: sig.GetValue("Subject", BsonNull.Value).AsString,
|
||||
KeyId: sig.GetValue("KeyId", BsonNull.Value).AsString,
|
||||
VerifiedAt: sig.Contains("VerifiedAt") && !sig["VerifiedAt"].IsBsonNull
|
||||
? new DateTimeOffset(sig["VerifiedAt"].ToUniversalTime(), TimeSpan.Zero)
|
||||
: null,
|
||||
TransparencyLogRef: sig.GetValue("TransparencyLogReference", BsonNull.Value).AsString,
|
||||
TrustWeight: sig.Contains("TrustWeight") && !sig["TrustWeight"].IsBsonNull
|
||||
? (decimal)sig["TrustWeight"].ToDouble()
|
||||
: null,
|
||||
TrustTier: DeriveTrustTier(sig.GetValue("TrustIssuerId", BsonNull.Value).AsString));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var productDoc = doc.GetValue("Product", BsonNull.Value).IsBsonDocument
|
||||
? doc["Product"].AsBsonDocument
|
||||
: null;
|
||||
|
||||
var product = new VexAdvisoryProductResponse(
|
||||
Key: productDoc?.GetValue("Key", BsonNull.Value).AsString ?? string.Empty,
|
||||
Name: productDoc?.GetValue("Name", BsonNull.Value).AsString,
|
||||
Version: productDoc?.GetValue("Version", BsonNull.Value).AsString,
|
||||
Purl: productDoc?.GetValue("Purl", BsonNull.Value).AsString,
|
||||
Cpe: productDoc?.GetValue("Cpe", BsonNull.Value).AsString);
|
||||
|
||||
statements.Add(new VexAdvisoryStatementResponse(
|
||||
StatementId: doc.GetValue("_id", BsonNull.Value).ToString() ?? string.Empty,
|
||||
ProviderId: doc.GetValue("ProviderId", BsonNull.Value).AsString ?? string.Empty,
|
||||
Product: product,
|
||||
Status: doc.GetValue("Status", BsonNull.Value).AsString ?? "unknown",
|
||||
Justification: doc.GetValue("Justification", BsonNull.Value).AsString,
|
||||
Detail: doc.GetValue("Detail", BsonNull.Value).AsString,
|
||||
FirstSeen: doc.GetValue("FirstSeen", BsonNull.Value).IsBsonDateTime
|
||||
? new DateTimeOffset(doc["FirstSeen"].ToUniversalTime(), TimeSpan.Zero)
|
||||
: now,
|
||||
LastSeen: doc.GetValue("LastSeen", BsonNull.Value).IsBsonDateTime
|
||||
? new DateTimeOffset(doc["LastSeen"].ToUniversalTime(), TimeSpan.Zero)
|
||||
: now,
|
||||
Provenance: provenance,
|
||||
Attestation: attestation));
|
||||
}
|
||||
|
||||
var aliases = canonicalKey.Links
|
||||
.Select(link => new VexAdvisoryLinkResponse(link.Identifier, link.Type, link.IsOriginal))
|
||||
.ToList();
|
||||
|
||||
stopwatch.Stop();
|
||||
NormalizationTelemetry.RecordEvidenceRetrieval(
|
||||
tenant,
|
||||
"success",
|
||||
statements.Count,
|
||||
stopwatch.Elapsed.TotalSeconds);
|
||||
|
||||
var response = new VexAdvisoryEvidenceResponse(
|
||||
AdvisoryKey: advisory_key.Trim(),
|
||||
CanonicalKey: canonicalKey.AdvisoryKey,
|
||||
Scope: canonicalKey.Scope.ToString().ToLowerInvariant(),
|
||||
Aliases: aliases,
|
||||
Statements: statements,
|
||||
QueriedAt: now,
|
||||
TotalCount: statements.Count);
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("GetVexAdvisoryEvidence");
|
||||
|
||||
// GET /evidence/vex/locker/{bundleId}
|
||||
app.MapGet("/evidence/vex/locker/{bundleId}", async (
|
||||
HttpContext context,
|
||||
string bundleId,
|
||||
[FromQuery] string? generation,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
IOptions<AirgapOptions> airgapOptions,
|
||||
[FromServices] IAirgapImportStore airgapImportStore,
|
||||
[FromServices] IVexHashingService hashingService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(bundleId))
|
||||
{
|
||||
return Results.BadRequest(new { error = new { code = "ERR_BUNDLE_ID", message = "bundleId is required" } });
|
||||
}
|
||||
|
||||
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (record is null)
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Locker manifest not found" } });
|
||||
}
|
||||
|
||||
// Optional local hash/size computation when locker root is configured
|
||||
long? manifestSize = null;
|
||||
long? evidenceSize = null;
|
||||
string? evidenceHash = null;
|
||||
|
||||
var lockerRoot = airgapOptions.Value.LockerRootPath;
|
||||
if (!string.IsNullOrWhiteSpace(lockerRoot))
|
||||
{
|
||||
TryHashFile(lockerRoot, record.PortableManifestPath, hashingService, out var manifestHash, out manifestSize);
|
||||
if (!string.IsNullOrWhiteSpace(manifestHash))
|
||||
{
|
||||
record.PortableManifestHash = manifestHash!;
|
||||
}
|
||||
|
||||
TryHashFile(lockerRoot, record.EvidenceLockerPath, hashingService, out evidenceHash, out evidenceSize);
|
||||
}
|
||||
|
||||
var timeline = record.Timeline
|
||||
.OrderBy(entry => entry.CreatedAt)
|
||||
.Select(entry => new VexEvidenceLockerTimelineEntry(
|
||||
entry.EventType,
|
||||
entry.CreatedAt,
|
||||
entry.ErrorCode,
|
||||
entry.Message,
|
||||
entry.StalenessSeconds))
|
||||
.ToList();
|
||||
|
||||
var response = new VexEvidenceLockerResponse(
|
||||
record.BundleId,
|
||||
record.MirrorGeneration,
|
||||
record.TenantId,
|
||||
record.Publisher,
|
||||
record.PayloadHash,
|
||||
record.PortableManifestPath,
|
||||
record.PortableManifestHash,
|
||||
record.EvidenceLockerPath,
|
||||
evidenceHash,
|
||||
manifestSize,
|
||||
evidenceSize,
|
||||
record.ImportedAt,
|
||||
record.Timeline.FirstOrDefault()?.StalenessSeconds,
|
||||
record.TransparencyLog,
|
||||
timeline);
|
||||
|
||||
return Results.Ok(response);
|
||||
}).WithName("GetVexEvidenceLockerManifest");
|
||||
|
||||
// GET /evidence/vex/locker/{bundleId}/manifest/file
|
||||
app.MapGet("/evidence/vex/locker/{bundleId}/manifest/file", async (
|
||||
HttpContext context,
|
||||
string bundleId,
|
||||
[FromQuery] string? generation,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
IOptions<AirgapOptions> airgapOptions,
|
||||
[FromServices] IAirgapImportStore airgapImportStore,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var root = airgapOptions.Value.LockerRootPath;
|
||||
if (string.IsNullOrWhiteSpace(root))
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_LOCKER_ROOT", message = "LockerRootPath is not configured" } });
|
||||
}
|
||||
|
||||
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (record is null)
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Locker manifest not found" } });
|
||||
}
|
||||
|
||||
if (!TryResolveLockerFile(root, record.PortableManifestPath, out var fullPath))
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_MANIFEST_FILE", message = "Manifest file not available" } });
|
||||
}
|
||||
|
||||
var (digest, size) = ComputeFileHash(fullPath);
|
||||
// Quote the ETag so HttpClient parses it into response.Headers.ETag.
|
||||
context.Response.Headers.ETag = $"\"{digest}\"";
|
||||
context.Response.ContentType = "application/json";
|
||||
context.Response.ContentLength = size;
|
||||
return Results.File(fullPath, "application/json");
|
||||
}).WithName("GetVexEvidenceLockerManifestFile");
|
||||
|
||||
// GET /evidence/vex/locker/{bundleId}/evidence/file
|
||||
app.MapGet("/evidence/vex/locker/{bundleId}/evidence/file", async (
|
||||
HttpContext context,
|
||||
string bundleId,
|
||||
[FromQuery] string? generation,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
IOptions<AirgapOptions> airgapOptions,
|
||||
[FromServices] IAirgapImportStore airgapImportStore,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
|
||||
if (scopeResult is not null)
|
||||
{
|
||||
return scopeResult;
|
||||
}
|
||||
|
||||
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var root = airgapOptions.Value.LockerRootPath;
|
||||
if (string.IsNullOrWhiteSpace(root))
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_LOCKER_ROOT", message = "LockerRootPath is not configured" } });
|
||||
}
|
||||
|
||||
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (record is null)
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Evidence file not found" } });
|
||||
}
|
||||
|
||||
if (!TryResolveLockerFile(root, record.EvidenceLockerPath, out var fullPath))
|
||||
{
|
||||
return Results.NotFound(new { error = new { code = "ERR_EVIDENCE_FILE", message = "Evidence file not available" } });
|
||||
}
|
||||
|
||||
var (digest, size) = ComputeFileHash(fullPath);
|
||||
// Quote the ETag so HttpClient parses it into response.Headers.ETag.
|
||||
context.Response.Headers.ETag = $"\"{digest}\"";
|
||||
context.Response.ContentType = "application/x-ndjson";
|
||||
context.Response.ContentLength = size;
|
||||
return Results.File(fullPath, "application/x-ndjson");
|
||||
}).WithName("GetVexEvidenceLockerEvidenceFile");
|
||||
}
|
||||
|
||||
private static void TryHashFile(string root, string relativePath, IVexHashingService hashingService, out string? digest, out long? size)
|
||||
{
|
||||
digest = null;
|
||||
size = null;
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!TryResolveLockerFile(root, relativePath, out var fullPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var data = File.ReadAllBytes(fullPath);
|
||||
digest = hashingService.ComputeHash(data, "sha256");
|
||||
size = data.LongLength;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore I/O errors and continue with stored metadata
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryResolveLockerFile(string root, string relativePath, out string fullPath)
|
||||
{
|
||||
fullPath = string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(root) || string.IsNullOrWhiteSpace(relativePath))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var rootFull = Path.GetFullPath(root);
|
||||
var candidate = Path.GetFullPath(Path.Combine(rootFull, relativePath));
|
||||
|
||||
if (!candidate.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!File.Exists(candidate))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
fullPath = candidate;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static (string Digest, long SizeBytes) ComputeFileHash(string path)
|
||||
{
|
||||
using var stream = File.OpenRead(path);
|
||||
using var sha = SHA256.Create();
|
||||
var hashBytes = sha.ComputeHash(stream);
|
||||
var digest = "sha256:" + Convert.ToHexString(hashBytes).ToLowerInvariant();
|
||||
var size = new FileInfo(path).Length;
|
||||
return (digest, size);
|
||||
}
|
||||
|
||||
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, out string tenant, out IResult? problem)
|
||||
{
|
||||
tenant = options.DefaultTenant;
|
||||
problem = null;
|
||||
|
||||
if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerValues) && headerValues.Count > 0)
|
||||
{
|
||||
var requestedTenant = headerValues[0]?.Trim();
|
||||
if (string.IsNullOrEmpty(requestedTenant))
|
||||
{
|
||||
problem = Results.BadRequest(new { error = new { code = "ERR_TENANT", message = "X-Stella-Tenant header must not be empty" } });
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!string.Equals(requestedTenant, options.DefaultTenant, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
problem = Results.Json(
|
||||
new { error = new { code = "ERR_TENANT_FORBIDDEN", message = $"Tenant '{requestedTenant}' is not allowed" } },
|
||||
statusCode: StatusCodes.Status403Forbidden);
|
||||
return false;
|
||||
}
|
||||
|
||||
tenant = requestedTenant;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryDecodeCursor(string cursor, out DateTime timestamp, out string id)
|
||||
{
|
||||
timestamp = default;
|
||||
id = string.Empty;
|
||||
try
|
||||
{
|
||||
var payload = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(cursor));
|
||||
var parts = payload.Split('|');
|
||||
if (parts.Length != 2)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParse(parts[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
timestamp = parsed.UtcDateTime;
|
||||
id = parts[1];
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string EncodeCursor(DateTime timestamp, string id)
|
||||
{
|
||||
var payload = FormattableString.Invariant($"{timestamp:O}|{id}");
|
||||
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(payload));
|
||||
}
|
||||
|
||||
private static string EscapeRegex(string input)
|
||||
{
|
||||
// Escape special regex characters for safe use in MongoDB regex
|
||||
return System.Text.RegularExpressions.Regex.Escape(input);
|
||||
}
|
||||
|
||||
private static string? DeriveTrustTier(string? issuerId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(issuerId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var lowerIssuerId = issuerId.ToLowerInvariant();
|
||||
if (lowerIssuerId.Contains("vendor") || lowerIssuerId.Contains("upstream"))
|
||||
{
|
||||
return "vendor";
|
||||
}
|
||||
|
||||
if (lowerIssuerId.Contains("distro") || lowerIssuerId.Contains("rhel") ||
|
||||
lowerIssuerId.Contains("ubuntu") || lowerIssuerId.Contains("debian"))
|
||||
{
|
||||
return "distro-trusted";
|
||||
}
|
||||
|
||||
if (lowerIssuerId.Contains("community") || lowerIssuerId.Contains("oss"))
|
||||
{
|
||||
return "community";
|
||||
}
|
||||
|
||||
return "other";
|
||||
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
|
||||
return Results.Problem(
|
||||
detail: "Evidence chunk streaming is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable,
|
||||
title: "Service unavailable");
|
||||
}).WithName("GetVexEvidenceChunks");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,6 +48,9 @@ services.AddOptions<VexStorageOptions>()
|
||||
.ValidateOnStart();
|
||||
|
||||
services.AddExcititorPostgresStorage(configuration);
|
||||
services.TryAddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
|
||||
services.TryAddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.TryAddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
|
||||
services.AddCsafNormalizer();
|
||||
services.AddCycloneDxNormalizer();
|
||||
services.AddOpenVexNormalizer();
|
||||
@@ -146,13 +149,12 @@ app.UseObservabilityHeaders();
|
||||
|
||||
app.MapGet("/excititor/status", async (HttpContext context,
|
||||
IEnumerable<IVexArtifactStore> artifactStores,
|
||||
IOptions<VexStorageOptions> mongoOptions,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
TimeProvider timeProvider) =>
|
||||
{
|
||||
var payload = new StatusResponse(
|
||||
timeProvider.GetUtcNow(),
|
||||
mongoOptions.Value.RawBucketName,
|
||||
mongoOptions.Value.GridFsInlineThresholdBytes,
|
||||
storageOptions.Value.InlineThresholdBytes,
|
||||
artifactStores.Select(store => store.GetType().Name).ToArray());
|
||||
|
||||
context.Response.ContentType = "application/json";
|
||||
@@ -210,19 +212,18 @@ app.MapGet("/openapi/excititor.json", () =>
|
||||
{
|
||||
schema = new { @ref = "#/components/schemas/StatusResponse" },
|
||||
examples = new Dictionary<string, object>
|
||||
{
|
||||
["example"] = new
|
||||
{
|
||||
value = new
|
||||
{
|
||||
timeUtc = "2025-11-24T00:00:00Z",
|
||||
mongoBucket = "vex-raw",
|
||||
gridFsInlineThresholdBytes = 1048576,
|
||||
artifactStores = new[] { "S3ArtifactStore", "OfflineBundleArtifactStore" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
["example"] = new
|
||||
{
|
||||
value = new
|
||||
{
|
||||
timeUtc = "2025-11-24T00:00:00Z",
|
||||
inlineThreshold = 1048576,
|
||||
artifactStores = new[] { "S3ArtifactStore", "OfflineBundleArtifactStore" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -892,12 +893,11 @@ app.MapGet("/openapi/excititor.json", () =>
|
||||
["StatusResponse"] = new
|
||||
{
|
||||
type = "object",
|
||||
required = new[] { "timeUtc", "mongoBucket", "artifactStores" },
|
||||
required = new[] { "timeUtc", "artifactStores", "inlineThreshold" },
|
||||
properties = new Dictionary<string, object>
|
||||
{
|
||||
["timeUtc"] = new { type = "string", format = "date-time" },
|
||||
["mongoBucket"] = new { type = "string" },
|
||||
["gridFsInlineThresholdBytes"] = new { type = "integer", format = "int64" },
|
||||
["inlineThreshold"] = new { type = "integer", format = "int64" },
|
||||
["artifactStores"] = new { type = "array", items = new { type = "string" } }
|
||||
}
|
||||
},
|
||||
@@ -2270,7 +2270,7 @@ internal sealed record ExcititorTimelineEvent(
|
||||
|
||||
public partial class Program;
|
||||
|
||||
internal sealed record StatusResponse(DateTimeOffset UtcNow, string MongoBucket, int InlineThreshold, string[] ArtifactStores);
|
||||
internal sealed record StatusResponse(DateTimeOffset UtcNow, int InlineThreshold, string[] ArtifactStores);
|
||||
|
||||
internal sealed record VexStatementIngestRequest(IReadOnlyList<VexStatementEntry> Statements);
|
||||
|
||||
|
||||
@@ -1,48 +1,49 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Services;
|
||||
|
||||
internal sealed class ExcititorHealthService
|
||||
{
|
||||
private const string RetrievedAtField = "RetrievedAt";
|
||||
private const string MetadataField = "Metadata";
|
||||
private const string CalculatedAtField = "CalculatedAt";
|
||||
private const string ConflictsField = "Conflicts";
|
||||
private const string ConflictStatusField = "Status";
|
||||
|
||||
private readonly IMongoDatabase _database;
|
||||
private readonly IVexRawStore _rawStore;
|
||||
private readonly IVexLinksetStore _linksetStore;
|
||||
private readonly IVexProviderStore _providerStore;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IReadOnlyDictionary<string, VexConnectorDescriptor> _connectors;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ExcititorObservabilityOptions _options;
|
||||
private readonly ILogger<ExcititorHealthService> _logger;
|
||||
private readonly string _defaultTenant;
|
||||
|
||||
public ExcititorHealthService(
|
||||
IMongoDatabase database,
|
||||
IVexRawStore rawStore,
|
||||
IVexLinksetStore linksetStore,
|
||||
IVexProviderStore providerStore,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IEnumerable<IVexConnector> connectors,
|
||||
TimeProvider timeProvider,
|
||||
IOptions<ExcititorObservabilityOptions> options,
|
||||
IOptions<VexStorageOptions> storageOptions,
|
||||
ILogger<ExcititorHealthService> logger)
|
||||
{
|
||||
_database = database ?? throw new ArgumentNullException(nameof(database));
|
||||
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
|
||||
_linksetStore = linksetStore ?? throw new ArgumentNullException(nameof(linksetStore));
|
||||
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_options = options?.Value ?? new ExcititorObservabilityOptions();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
var storage = storageOptions?.Value ?? new VexStorageOptions();
|
||||
_defaultTenant = string.IsNullOrWhiteSpace(storage.DefaultTenant)
|
||||
? "default"
|
||||
: storage.DefaultTenant.Trim();
|
||||
|
||||
if (connectors is null)
|
||||
{
|
||||
@@ -158,7 +159,7 @@ internal sealed class ExcititorHealthService
|
||||
private LinkHealthSection BuildLinkSection(DateTimeOffset now, LinkSnapshot snapshot)
|
||||
{
|
||||
TimeSpan? lag = null;
|
||||
if (snapshot.LastConsensusAt is { } calculatedAt)
|
||||
if (snapshot.LastUpdatedAt is { } calculatedAt)
|
||||
{
|
||||
lag = now - calculatedAt;
|
||||
if (lag < TimeSpan.Zero)
|
||||
@@ -174,7 +175,7 @@ internal sealed class ExcititorHealthService
|
||||
|
||||
return new LinkHealthSection(
|
||||
status,
|
||||
snapshot.LastConsensusAt,
|
||||
snapshot.LastUpdatedAt,
|
||||
lag?.TotalSeconds,
|
||||
snapshot.TotalDocuments,
|
||||
snapshot.DocumentsWithConflicts);
|
||||
@@ -271,47 +272,36 @@ internal sealed class ExcititorHealthService
|
||||
var window = _options.GetPositive(_options.SignatureWindow, TimeSpan.FromHours(12));
|
||||
var windowStart = now - window;
|
||||
|
||||
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var filter = Builders<BsonDocument>.Filter.Gte(RetrievedAtField, windowStart.UtcDateTime);
|
||||
var projection = Builders<BsonDocument>.Projection
|
||||
.Include(MetadataField)
|
||||
.Include(RetrievedAtField);
|
||||
|
||||
List<BsonDocument> documents;
|
||||
try
|
||||
{
|
||||
documents = await collection
|
||||
.Find(filter)
|
||||
.Project(projection)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load signature window metrics.");
|
||||
documents = new List<BsonDocument>();
|
||||
}
|
||||
var page = await _rawStore.QueryAsync(
|
||||
new VexRawQuery(
|
||||
_defaultTenant,
|
||||
Array.Empty<string>(),
|
||||
Array.Empty<string>(),
|
||||
Array.Empty<VexDocumentFormat>(),
|
||||
windowStart,
|
||||
until: null,
|
||||
Cursor: null,
|
||||
Limit: 500),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var evaluated = 0;
|
||||
var withSignatures = 0;
|
||||
var verified = 0;
|
||||
|
||||
foreach (var document in documents)
|
||||
foreach (var document in page.Items)
|
||||
{
|
||||
evaluated++;
|
||||
if (!document.TryGetValue(MetadataField, out var metadataValue) ||
|
||||
metadataValue is not BsonDocument metadata ||
|
||||
metadata.ElementCount == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryGetBoolean(metadata, "signature.present", out var present) && present)
|
||||
var metadata = document.Metadata;
|
||||
if (metadata.TryGetValue("signature.present", out var presentValue) &&
|
||||
bool.TryParse(presentValue, out var present) &&
|
||||
present)
|
||||
{
|
||||
withSignatures++;
|
||||
}
|
||||
|
||||
if (TryGetBoolean(metadata, "signature.verified", out var verifiedFlag) && verifiedFlag)
|
||||
if (metadata.TryGetValue("signature.verified", out var verifiedValue) &&
|
||||
bool.TryParse(verifiedValue, out var verifiedFlag) &&
|
||||
verifiedFlag)
|
||||
{
|
||||
verified++;
|
||||
}
|
||||
@@ -322,80 +312,43 @@ internal sealed class ExcititorHealthService
|
||||
|
||||
private async Task<LinkSnapshot> LoadLinkSnapshotAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
|
||||
|
||||
BsonDocument? latest = null;
|
||||
try
|
||||
{
|
||||
latest = await collection
|
||||
.Find(Builders<BsonDocument>.Filter.Empty)
|
||||
.Sort(Builders<BsonDocument>.Sort.Descending(CalculatedAtField))
|
||||
.Project(Builders<BsonDocument>.Projection.Include(CalculatedAtField))
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read latest consensus document.");
|
||||
}
|
||||
|
||||
DateTimeOffset? lastConsensusAt = null;
|
||||
if (latest is not null &&
|
||||
latest.TryGetValue(CalculatedAtField, out var dateValue))
|
||||
{
|
||||
var utc = TryReadDateTime(dateValue);
|
||||
if (utc is not null)
|
||||
{
|
||||
lastConsensusAt = new DateTimeOffset(utc.Value, TimeSpan.Zero);
|
||||
}
|
||||
}
|
||||
|
||||
long totalDocuments = 0;
|
||||
long conflictDocuments = 0;
|
||||
DateTimeOffset? lastUpdated = null;
|
||||
|
||||
try
|
||||
{
|
||||
totalDocuments = await collection.EstimatedDocumentCountAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
conflictDocuments = await collection.CountDocumentsAsync(
|
||||
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"),
|
||||
cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
totalDocuments = await _linksetStore.CountAsync(_defaultTenant, cancellationToken).ConfigureAwait(false);
|
||||
conflictDocuments = await _linksetStore.CountWithConflictsAsync(_defaultTenant, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var conflictSample = await _linksetStore.FindWithConflictsAsync(_defaultTenant, 1, cancellationToken).ConfigureAwait(false);
|
||||
if (conflictSample.Count > 0)
|
||||
{
|
||||
lastUpdated = conflictSample[0].UpdatedAt;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to compute consensus counts.");
|
||||
_logger.LogWarning(ex, "Failed to compute linkset counts.");
|
||||
}
|
||||
|
||||
return new LinkSnapshot(lastConsensusAt, totalDocuments, conflictDocuments);
|
||||
return new LinkSnapshot(lastUpdated, totalDocuments, conflictDocuments);
|
||||
}
|
||||
|
||||
private async Task<ConflictSnapshot> LoadConflictSnapshotAsync(DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
var window = _options.GetPositive(_options.ConflictTrendWindow, TimeSpan.FromHours(24));
|
||||
var windowStart = now - window;
|
||||
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
|
||||
|
||||
var filter = Builders<BsonDocument>.Filter.And(
|
||||
Builders<BsonDocument>.Filter.Gte(CalculatedAtField, windowStart.UtcDateTime),
|
||||
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"));
|
||||
|
||||
var projection = Builders<BsonDocument>.Projection
|
||||
.Include(CalculatedAtField)
|
||||
.Include(ConflictsField);
|
||||
|
||||
List<BsonDocument> documents;
|
||||
IReadOnlyList<VexLinkset> linksets;
|
||||
try
|
||||
{
|
||||
documents = await collection
|
||||
.Find(filter)
|
||||
.Project(projection)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
// Sample conflicted linksets (ordered by updated_at DESC in Postgres implementation)
|
||||
linksets = await _linksetStore.FindWithConflictsAsync(_defaultTenant, 500, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load conflict trend window.");
|
||||
documents = new List<BsonDocument>();
|
||||
linksets = Array.Empty<VexLinkset>();
|
||||
}
|
||||
|
||||
var byStatus = new Dictionary<string, long>(StringComparer.OrdinalIgnoreCase);
|
||||
@@ -405,47 +358,31 @@ internal sealed class ExcititorHealthService
|
||||
var bucketMinutes = Math.Max(1, _options.ConflictTrendBucketMinutes);
|
||||
var bucketTicks = TimeSpan.FromMinutes(bucketMinutes).Ticks;
|
||||
|
||||
foreach (var doc in documents)
|
||||
foreach (var linkset in linksets)
|
||||
{
|
||||
if (!doc.TryGetValue(ConflictsField, out var conflictsValue) ||
|
||||
conflictsValue is not BsonArray conflicts ||
|
||||
conflicts.Count == 0)
|
||||
if (linkset.Disagreements.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
docsWithConflicts++;
|
||||
totalConflicts += conflicts.Count;
|
||||
totalConflicts += linkset.Disagreements.Count;
|
||||
|
||||
foreach (var conflictValue in conflicts.OfType<BsonDocument>())
|
||||
foreach (var disagreement in linkset.Disagreements)
|
||||
{
|
||||
var status = conflictValue.TryGetValue(ConflictStatusField, out var statusValue) && statusValue.IsString
|
||||
? statusValue.AsString
|
||||
: "unknown";
|
||||
|
||||
if (string.IsNullOrWhiteSpace(status))
|
||||
{
|
||||
status = "unknown";
|
||||
}
|
||||
var status = string.IsNullOrWhiteSpace(disagreement.Status)
|
||||
? "unknown"
|
||||
: disagreement.Status;
|
||||
|
||||
byStatus[status] = byStatus.TryGetValue(status, out var current)
|
||||
? current + 1
|
||||
: 1;
|
||||
}
|
||||
|
||||
if (doc.TryGetValue(CalculatedAtField, out var calculatedValue))
|
||||
{
|
||||
var utc = TryReadDateTime(calculatedValue);
|
||||
if (utc is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var alignedTicks = AlignTicks(utc.Value, bucketTicks);
|
||||
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var current)
|
||||
? current + conflicts.Count
|
||||
: conflicts.Count;
|
||||
}
|
||||
var alignedTicks = AlignTicks(linkset.UpdatedAt.UtcDateTime, bucketTicks);
|
||||
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var currentCount)
|
||||
? currentCount + linkset.Disagreements.Count
|
||||
: linkset.Disagreements.Count;
|
||||
}
|
||||
|
||||
var trend = timeline
|
||||
@@ -541,54 +478,6 @@ internal sealed class ExcititorHealthService
|
||||
return ticks - (ticks % bucketTicks);
|
||||
}
|
||||
|
||||
private static DateTime? TryReadDateTime(BsonValue value)
|
||||
{
|
||||
if (value is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value.IsBsonDateTime)
|
||||
{
|
||||
return value.AsBsonDateTime.ToUniversalTime();
|
||||
}
|
||||
|
||||
if (value.IsString &&
|
||||
DateTime.TryParse(
|
||||
value.AsString,
|
||||
CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
|
||||
out var parsed))
|
||||
{
|
||||
return DateTime.SpecifyKind(parsed, DateTimeKind.Utc);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryGetBoolean(BsonDocument document, string key, out bool value)
|
||||
{
|
||||
value = default;
|
||||
if (!document.TryGetValue(key, out var bsonValue))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (bsonValue.IsBoolean)
|
||||
{
|
||||
value = bsonValue.AsBoolean;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (bsonValue.IsString && bool.TryParse(bsonValue.AsString, out var parsed))
|
||||
{
|
||||
value = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static VexConnectorDescriptor DescribeConnector(IVexConnector connector)
|
||||
=> connector switch
|
||||
{
|
||||
@@ -596,7 +485,7 @@ internal sealed class ExcititorHealthService
|
||||
_ => new VexConnectorDescriptor(connector.Id, connector.Kind, connector.Id)
|
||||
};
|
||||
|
||||
private sealed record LinkSnapshot(DateTimeOffset? LastConsensusAt, long TotalDocuments, long DocumentsWithConflicts);
|
||||
private sealed record LinkSnapshot(DateTimeOffset? LastUpdatedAt, long TotalDocuments, long DocumentsWithConflicts);
|
||||
|
||||
private sealed record ConflictSnapshot(
|
||||
DateTimeOffset WindowStart,
|
||||
|
||||
@@ -5,7 +5,6 @@ using System.Globalization;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
@@ -151,7 +150,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
var result = await ExecuteRunAsync(runId, handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
|
||||
var result = await ExecuteRunAsync(runId, handle, since, options.Force, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
@@ -174,8 +173,8 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
|
||||
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
|
||||
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, cancellationToken).ConfigureAwait(false);
|
||||
var result = await ExecuteRunAsync(runId, handle, since, force: false, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
@@ -201,14 +200,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
{
|
||||
try
|
||||
{
|
||||
var state = await _stateRepository.GetAsync(handle.Descriptor.Id, cancellationToken, session).ConfigureAwait(false);
|
||||
var state = await _stateRepository.GetAsync(handle.Descriptor.Id, cancellationToken).ConfigureAwait(false);
|
||||
var lastUpdated = state?.LastUpdated;
|
||||
var stale = threshold.HasValue && (lastUpdated is null || lastUpdated < threshold.Value);
|
||||
|
||||
if (stale || state is null)
|
||||
{
|
||||
var since = stale ? threshold : lastUpdated;
|
||||
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
|
||||
var result = await ExecuteRunAsync(runId, handle, since, force: false, cancellationToken).ConfigureAwait(false);
|
||||
results.Add(new ReconcileProviderResult(
|
||||
handle.Descriptor.Id,
|
||||
result.Status,
|
||||
@@ -271,14 +270,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
|
||||
private async Task EnsureProviderRegistrationAsync(VexConnectorDescriptor descriptor, CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false);
|
||||
var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var provider = new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
|
||||
await _providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
|
||||
await _providerStore.SaveAsync(provider, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<ProviderRunResult> ExecuteRunAsync(
|
||||
@@ -286,7 +285,6 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
ConnectorHandle handle,
|
||||
DateTimeOffset? since,
|
||||
bool force,
|
||||
IClientSessionHandle session,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var providerId = handle.Descriptor.Id;
|
||||
@@ -304,15 +302,15 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
try
|
||||
{
|
||||
await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureProviderRegistrationAsync(handle.Descriptor, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (force)
|
||||
{
|
||||
var resetState = new VexConnectorState(providerId, null, ImmutableArray<string>.Empty);
|
||||
await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false);
|
||||
await _stateRepository.SaveAsync(resetState, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
|
||||
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
|
||||
var resumeTokens = stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
var context = new VexConnectorContext(
|
||||
@@ -337,13 +335,13 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0)
|
||||
{
|
||||
claims += batch.Claims.Length;
|
||||
await _claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false);
|
||||
await _claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
var completedAt = _timeProvider.GetUtcNow();
|
||||
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
|
||||
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var checkpoint = stateAfterRun?.DocumentDigests.IsDefaultOrEmpty == false
|
||||
? stateAfterRun.DocumentDigests[^1]
|
||||
@@ -413,7 +411,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DateTimeOffset?> ResolveResumeSinceAsync(string providerId, string? checkpoint, IClientSessionHandle session, CancellationToken cancellationToken)
|
||||
private async Task<DateTimeOffset?> ResolveResumeSinceAsync(string providerId, string? checkpoint, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(checkpoint))
|
||||
{
|
||||
@@ -427,14 +425,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
|
||||
}
|
||||
|
||||
var digest = checkpoint.Trim();
|
||||
var document = await _rawStore.FindByDigestAsync(digest, cancellationToken, session).ConfigureAwait(false);
|
||||
var document = await _rawStore.FindByDigestAsync(digest, cancellationToken).ConfigureAwait(false);
|
||||
if (document is not null)
|
||||
{
|
||||
return document.RetrievedAt;
|
||||
}
|
||||
}
|
||||
|
||||
var state = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
|
||||
var state = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
|
||||
return state?.LastUpdated;
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />
|
||||
|
||||
@@ -30,7 +30,7 @@ Run Excititor background jobs (ingestion, linkset extraction, dedup/idempotency
|
||||
- Keep timestamps UTC ISO-8601; inject clock/GUID providers for tests.
|
||||
|
||||
## Boundaries
|
||||
- Delegate domain logic to Core and persistence to Storage.Mongo; avoid embedding policy or UI concerns.
|
||||
- Delegate domain logic to Core and persistence to Storage.Postgres; avoid embedding policy or UI concerns.
|
||||
- Configuration via appsettings/environment; no hard-coded secrets.
|
||||
|
||||
## Ready-to-Start Checklist
|
||||
|
||||
@@ -12,7 +12,6 @@ using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Orchestration;
|
||||
|
||||
@@ -8,11 +8,12 @@ using StellaOps.Plugin;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Formats.CSAF;
|
||||
using StellaOps.Excititor.Formats.CycloneDX;
|
||||
using StellaOps.Excititor.Formats.OpenVEX;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Storage.Postgres;
|
||||
using StellaOps.Excititor.Worker.Auth;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
using StellaOps.Excititor.Worker.Orchestration;
|
||||
@@ -43,11 +44,14 @@ services.PostConfigure<VexWorkerOptions>(options =>
|
||||
});
|
||||
services.AddRedHatCsafConnector();
|
||||
|
||||
services.AddOptions<VexMongoStorageOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
|
||||
services.AddOptions<VexStorageOptions>()
|
||||
.Bind(configuration.GetSection("Excititor:Storage"))
|
||||
.ValidateOnStart();
|
||||
|
||||
services.AddExcititorMongoStorage();
|
||||
services.AddExcititorPostgresStorage(configuration);
|
||||
services.AddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
|
||||
services.AddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.AddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
|
||||
services.AddCsafNormalizer();
|
||||
services.AddCycloneDxNormalizer();
|
||||
services.AddOpenVexNormalizer();
|
||||
|
||||
@@ -5,12 +5,10 @@ using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Plugin;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
using StellaOps.Excititor.Worker.Orchestration;
|
||||
using StellaOps.Excititor.Worker.Signature;
|
||||
@@ -95,12 +93,6 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner
|
||||
var stateRepository = scopeProvider.GetRequiredService<IVexConnectorStateRepository>();
|
||||
var normalizerRouter = scopeProvider.GetRequiredService<IVexNormalizerRouter>();
|
||||
var signatureVerifier = scopeProvider.GetRequiredService<IVexSignatureVerifier>();
|
||||
var sessionProvider = scopeProvider.GetService<IVexMongoSessionProvider>();
|
||||
IClientSessionHandle? session = null;
|
||||
if (sessionProvider is not null)
|
||||
{
|
||||
session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var descriptor = connector switch
|
||||
{
|
||||
@@ -108,12 +100,12 @@ internal sealed class DefaultVexProviderRunner : IVexProviderRunner
|
||||
_ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id)
|
||||
};
|
||||
|
||||
var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false)
|
||||
var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken).ConfigureAwait(false)
|
||||
?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
|
||||
|
||||
await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
|
||||
await providerStore.SaveAsync(provider, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var stateBeforeRun = await stateRepository.GetAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false);
|
||||
var stateBeforeRun = await stateRepository.GetAsync(descriptor.Id, cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
if (stateBeforeRun?.NextEligibleRun is { } nextEligible && nextEligible > now)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,65 +1,64 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Signature;
|
||||
|
||||
internal sealed class VerifyingVexRawDocumentSink : IVexRawDocumentSink
|
||||
{
|
||||
private readonly IVexRawStore _inner;
|
||||
private readonly IVexSignatureVerifier _signatureVerifier;
|
||||
|
||||
public VerifyingVexRawDocumentSink(IVexRawStore inner, IVexSignatureVerifier signatureVerifier)
|
||||
{
|
||||
_inner = inner ?? throw new ArgumentNullException(nameof(inner));
|
||||
_signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier));
|
||||
}
|
||||
|
||||
public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var signatureMetadata = await _signatureVerifier.VerifyAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
var enrichedDocument = signatureMetadata is null
|
||||
? document
|
||||
: document with { Metadata = EnrichMetadata(document.Metadata, signatureMetadata) };
|
||||
|
||||
await _inner.StoreAsync(enrichedDocument, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static ImmutableDictionary<string, string> EnrichMetadata(
|
||||
ImmutableDictionary<string, string> metadata,
|
||||
VexSignatureMetadata signature)
|
||||
{
|
||||
var builder = metadata is null
|
||||
? ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal)
|
||||
: metadata.ToBuilder();
|
||||
|
||||
builder["signature.present"] = "true";
|
||||
builder["signature.verified"] = "true";
|
||||
builder["vex.signature.type"] = signature.Type;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.Subject))
|
||||
{
|
||||
builder["vex.signature.subject"] = signature.Subject!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.Issuer))
|
||||
{
|
||||
builder["vex.signature.issuer"] = signature.Issuer!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.KeyId))
|
||||
{
|
||||
builder["vex.signature.keyId"] = signature.KeyId!;
|
||||
}
|
||||
|
||||
if (signature.VerifiedAt is not null)
|
||||
{
|
||||
builder["vex.signature.verifiedAt"] = signature.VerifiedAt.Value.ToString("O");
|
||||
}
|
||||
|
||||
|
||||
internal sealed class VerifyingVexRawDocumentSink : IVexRawDocumentSink
|
||||
{
|
||||
private readonly IVexRawStore _inner;
|
||||
private readonly IVexSignatureVerifier _signatureVerifier;
|
||||
|
||||
public VerifyingVexRawDocumentSink(IVexRawStore inner, IVexSignatureVerifier signatureVerifier)
|
||||
{
|
||||
_inner = inner ?? throw new ArgumentNullException(nameof(inner));
|
||||
_signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier));
|
||||
}
|
||||
|
||||
public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var signatureMetadata = await _signatureVerifier.VerifyAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
var enrichedDocument = signatureMetadata is null
|
||||
? document
|
||||
: document with { Metadata = EnrichMetadata(document.Metadata, signatureMetadata) };
|
||||
|
||||
await _inner.StoreAsync(enrichedDocument, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static ImmutableDictionary<string, string> EnrichMetadata(
|
||||
ImmutableDictionary<string, string> metadata,
|
||||
VexSignatureMetadata signature)
|
||||
{
|
||||
var builder = metadata is null
|
||||
? ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal)
|
||||
: metadata.ToBuilder();
|
||||
|
||||
builder["signature.present"] = "true";
|
||||
builder["signature.verified"] = "true";
|
||||
builder["vex.signature.type"] = signature.Type;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.Subject))
|
||||
{
|
||||
builder["vex.signature.subject"] = signature.Subject!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.Issuer))
|
||||
{
|
||||
builder["vex.signature.issuer"] = signature.Issuer!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.KeyId))
|
||||
{
|
||||
builder["vex.signature.keyId"] = signature.KeyId!;
|
||||
}
|
||||
|
||||
if (signature.VerifiedAt is not null)
|
||||
{
|
||||
builder["vex.signature.verifiedAt"] = signature.VerifiedAt.Value.ToString("O");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(signature.TransparencyLogReference))
|
||||
{
|
||||
builder["vex.signature.transparencyLogReference"] = signature.TransparencyLogReference!;
|
||||
|
||||
@@ -14,12 +14,10 @@
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="../../Concelier/__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<!-- Temporarily commented out: RedHat CSAF connector blocked by missing Storage.Mongo project -->
|
||||
<!-- <ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj" /> -->
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />
|
||||
<!-- Temporarily commented out: Storage.Mongo project not found -->
|
||||
<!-- <ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" /> -->
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj" />
|
||||
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj" />
|
||||
|
||||
@@ -13,8 +13,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Aoc", "..\Aoc\__L
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.RawModels", "..\Concelier\__Libraries\StellaOps.Concelier.RawModels\StellaOps.Concelier.RawModels.csproj", "{2D19CC50-EFE9-4015-B4DB-6DFF4E41DB11}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Storage.Mongo", "__Libraries\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj", "{5858415D-8AB4-4E45-B316-580879FD8339}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Export", "__Libraries\StellaOps.Excititor.Export\StellaOps.Excititor.Export.csproj", "{E8B20DD0-9282-4DFD-B363-F0AF7F62AED5}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Policy", "__Libraries\StellaOps.Excititor.Policy\StellaOps.Excititor.Policy.csproj", "{400690F2-466B-4DF0-B495-9015DBBAA046}"
|
||||
@@ -85,10 +83,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Formats
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Policy.Tests", "__Tests\StellaOps.Excititor.Policy.Tests\StellaOps.Excititor.Policy.Tests.csproj", "{832F539E-17FC-46B4-9E67-39BE5131352D}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Storage.Mongo.Tests", "__Tests\StellaOps.Excititor.Storage.Mongo.Tests\StellaOps.Excititor.Storage.Mongo.Tests.csproj", "{5BB6E9E8-3470-4BFF-94DD-DA3294616C39}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Storage.Mongo", "..\Concelier\__Libraries\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj", "{6507860E-BF0D-4E32-A6AC-49E1CE15E4B7}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Core", "..\Concelier\__Libraries\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj", "{D6014A0A-6BF4-45C8-918E-9558A24AAC5B}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Models", "..\Concelier\__Libraries\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj", "{13AF13D1-84C3-4D4F-B89A-0653102C3E63}"
|
||||
|
||||
@@ -16,6 +16,7 @@ Connector responsible for ingesting Cisco CSAF VEX advisories and handing raw do
|
||||
## In/Out of scope
|
||||
In: data fetching, provider metadata, retry controls, raw document persistence.
|
||||
Out: normalization/export, attestation, Mongo wiring (handled in other modules).
|
||||
Out: normalization/export, attestation, Postgres/in-memory wiring (handled in other modules).
|
||||
## Observability & security expectations
|
||||
- Log fetch batches with document counts/durations; mask credentials.
|
||||
- Emit metrics for rate-limit hits, retries, and quarantine events.
|
||||
|
||||
@@ -11,68 +11,68 @@ using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Cisco.CSAF;
|
||||
|
||||
public sealed class CiscoCsafConnector : VexConnectorBase
|
||||
{
|
||||
private static readonly VexConnectorDescriptor DescriptorInstance = new(
|
||||
id: "excititor:cisco",
|
||||
kind: VexProviderKind.Vendor,
|
||||
displayName: "Cisco CSAF")
|
||||
{
|
||||
Tags = ImmutableArray.Create("cisco", "csaf"),
|
||||
};
|
||||
|
||||
private readonly CiscoProviderMetadataLoader _metadataLoader;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>> _validators;
|
||||
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private CiscoConnectorOptions? _options;
|
||||
private CiscoProviderMetadataResult? _providerMetadata;
|
||||
|
||||
public CiscoCsafConnector(
|
||||
CiscoProviderMetadataLoader metadataLoader,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>>? validators,
|
||||
ILogger<CiscoCsafConnector> logger,
|
||||
TimeProvider timeProvider)
|
||||
: base(DescriptorInstance, logger, timeProvider)
|
||||
{
|
||||
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<CiscoConnectorOptions>>();
|
||||
}
|
||||
|
||||
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
{
|
||||
_options = VexConnectorOptionsBinder.Bind(
|
||||
Descriptor,
|
||||
settings,
|
||||
validators: _validators);
|
||||
|
||||
_providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
|
||||
LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary<string, object?>
|
||||
{
|
||||
["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length,
|
||||
["fromOffline"] = _providerMetadata.FromOfflineSnapshot,
|
||||
});
|
||||
}
|
||||
|
||||
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
if (_options is null)
|
||||
{
|
||||
throw new InvalidOperationException("Connector must be validated before fetch operations.");
|
||||
}
|
||||
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Cisco.CSAF;
|
||||
|
||||
public sealed class CiscoCsafConnector : VexConnectorBase
|
||||
{
|
||||
private static readonly VexConnectorDescriptor DescriptorInstance = new(
|
||||
id: "excititor:cisco",
|
||||
kind: VexProviderKind.Vendor,
|
||||
displayName: "Cisco CSAF")
|
||||
{
|
||||
Tags = ImmutableArray.Create("cisco", "csaf"),
|
||||
};
|
||||
|
||||
private readonly CiscoProviderMetadataLoader _metadataLoader;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>> _validators;
|
||||
private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private CiscoConnectorOptions? _options;
|
||||
private CiscoProviderMetadataResult? _providerMetadata;
|
||||
|
||||
public CiscoCsafConnector(
|
||||
CiscoProviderMetadataLoader metadataLoader,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IEnumerable<IVexConnectorOptionsValidator<CiscoConnectorOptions>>? validators,
|
||||
ILogger<CiscoCsafConnector> logger,
|
||||
TimeProvider timeProvider)
|
||||
: base(DescriptorInstance, logger, timeProvider)
|
||||
{
|
||||
_metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader));
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<CiscoConnectorOptions>>();
|
||||
}
|
||||
|
||||
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
{
|
||||
_options = VexConnectorOptionsBinder.Bind(
|
||||
Descriptor,
|
||||
settings,
|
||||
validators: _validators);
|
||||
|
||||
_providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
|
||||
LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary<string, object?>
|
||||
{
|
||||
["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length,
|
||||
["fromOffline"] = _providerMetadata.FromOfflineSnapshot,
|
||||
});
|
||||
}
|
||||
|
||||
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
if (_options is null)
|
||||
{
|
||||
throw new InvalidOperationException("Connector must be validated before fetch operations.");
|
||||
}
|
||||
|
||||
if (_providerMetadata is null)
|
||||
{
|
||||
_providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false);
|
||||
@@ -81,28 +81,28 @@ public sealed class CiscoCsafConnector : VexConnectorBase
|
||||
await UpsertProviderAsync(context.Services, _providerMetadata.Provider, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
|
||||
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
|
||||
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
|
||||
var digestList = new List<string>(knownDigests);
|
||||
var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue;
|
||||
var latestTimestamp = state?.LastUpdated ?? since;
|
||||
var stateChanged = false;
|
||||
|
||||
var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName);
|
||||
foreach (var directory in _providerMetadata.Provider.BaseUris)
|
||||
{
|
||||
await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue;
|
||||
if (published <= since)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
contentResponse.EnsureSuccessStatusCode();
|
||||
var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
|
||||
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
|
||||
var digestList = new List<string>(knownDigests);
|
||||
var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue;
|
||||
var latestTimestamp = state?.LastUpdated ?? since;
|
||||
var stateChanged = false;
|
||||
|
||||
var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName);
|
||||
foreach (var directory in _providerMetadata.Provider.BaseUris)
|
||||
{
|
||||
await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue;
|
||||
if (published <= since)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
contentResponse.EnsureSuccessStatusCode();
|
||||
var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = BuildMetadata(builder =>
|
||||
{
|
||||
builder
|
||||
@@ -120,118 +120,118 @@ public sealed class CiscoCsafConnector : VexConnectorBase
|
||||
advisory.DocumentUri,
|
||||
payload,
|
||||
metadata);
|
||||
|
||||
if (!digestSet.Add(rawDocument.Digest))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
|
||||
digestList.Add(rawDocument.Digest);
|
||||
stateChanged = true;
|
||||
if (published > latestTimestamp)
|
||||
{
|
||||
latestTimestamp = published;
|
||||
}
|
||||
|
||||
yield return rawDocument;
|
||||
}
|
||||
}
|
||||
|
||||
if (stateChanged)
|
||||
{
|
||||
var baseState = state ?? new VexConnectorState(
|
||||
Descriptor.Id,
|
||||
null,
|
||||
ImmutableArray<string>.Empty,
|
||||
ImmutableDictionary<string, string>.Empty,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null);
|
||||
var newState = baseState with
|
||||
{
|
||||
LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp,
|
||||
DocumentDigests = digestList.ToImmutableArray(),
|
||||
};
|
||||
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing.");
|
||||
|
||||
private async IAsyncEnumerable<CiscoAdvisoryEntry> EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var nextUri = BuildIndexUri(directory, null);
|
||||
while (nextUri is not null)
|
||||
{
|
||||
using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
var page = JsonSerializer.Deserialize<CiscoAdvisoryIndex>(json, _serializerOptions);
|
||||
if (page?.Advisories is null)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var advisory in page.Advisories)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(advisory.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!documentUri.IsAbsoluteUri)
|
||||
{
|
||||
documentUri = new Uri(directory, documentUri);
|
||||
}
|
||||
|
||||
yield return new CiscoAdvisoryEntry(
|
||||
advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(),
|
||||
documentUri,
|
||||
advisory.Revision,
|
||||
advisory.Published,
|
||||
advisory.LastModified,
|
||||
advisory.Sha256);
|
||||
}
|
||||
|
||||
nextUri = ResolveNextUri(directory, page.Next);
|
||||
}
|
||||
}
|
||||
|
||||
private static Uri BuildIndexUri(Uri directory, string? relative)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relative))
|
||||
{
|
||||
var baseText = directory.ToString();
|
||||
if (!baseText.EndsWith('/'))
|
||||
{
|
||||
baseText += "/";
|
||||
}
|
||||
|
||||
return new Uri(new Uri(baseText, UriKind.Absolute), "index.json");
|
||||
}
|
||||
|
||||
if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute))
|
||||
{
|
||||
return absolute;
|
||||
}
|
||||
|
||||
var baseTextRelative = directory.ToString();
|
||||
if (!baseTextRelative.EndsWith('/'))
|
||||
{
|
||||
baseTextRelative += "/";
|
||||
}
|
||||
|
||||
return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative);
|
||||
}
|
||||
|
||||
|
||||
if (!digestSet.Add(rawDocument.Digest))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
|
||||
digestList.Add(rawDocument.Digest);
|
||||
stateChanged = true;
|
||||
if (published > latestTimestamp)
|
||||
{
|
||||
latestTimestamp = published;
|
||||
}
|
||||
|
||||
yield return rawDocument;
|
||||
}
|
||||
}
|
||||
|
||||
if (stateChanged)
|
||||
{
|
||||
var baseState = state ?? new VexConnectorState(
|
||||
Descriptor.Id,
|
||||
null,
|
||||
ImmutableArray<string>.Empty,
|
||||
ImmutableDictionary<string, string>.Empty,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null);
|
||||
var newState = baseState with
|
||||
{
|
||||
LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp,
|
||||
DocumentDigests = digestList.ToImmutableArray(),
|
||||
};
|
||||
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing.");
|
||||
|
||||
private async IAsyncEnumerable<CiscoAdvisoryEntry> EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
var nextUri = BuildIndexUri(directory, null);
|
||||
while (nextUri is not null)
|
||||
{
|
||||
using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
var page = JsonSerializer.Deserialize<CiscoAdvisoryIndex>(json, _serializerOptions);
|
||||
if (page?.Advisories is null)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
foreach (var advisory in page.Advisories)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(advisory.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!documentUri.IsAbsoluteUri)
|
||||
{
|
||||
documentUri = new Uri(directory, documentUri);
|
||||
}
|
||||
|
||||
yield return new CiscoAdvisoryEntry(
|
||||
advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(),
|
||||
documentUri,
|
||||
advisory.Revision,
|
||||
advisory.Published,
|
||||
advisory.LastModified,
|
||||
advisory.Sha256);
|
||||
}
|
||||
|
||||
nextUri = ResolveNextUri(directory, page.Next);
|
||||
}
|
||||
}
|
||||
|
||||
private static Uri BuildIndexUri(Uri directory, string? relative)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relative))
|
||||
{
|
||||
var baseText = directory.ToString();
|
||||
if (!baseText.EndsWith('/'))
|
||||
{
|
||||
baseText += "/";
|
||||
}
|
||||
|
||||
return new Uri(new Uri(baseText, UriKind.Absolute), "index.json");
|
||||
}
|
||||
|
||||
if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute))
|
||||
{
|
||||
return absolute;
|
||||
}
|
||||
|
||||
var baseTextRelative = directory.ToString();
|
||||
if (!baseTextRelative.EndsWith('/'))
|
||||
{
|
||||
baseTextRelative += "/";
|
||||
}
|
||||
|
||||
return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative);
|
||||
}
|
||||
|
||||
private static Uri? ResolveNextUri(Uri directory, string? next)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(next))
|
||||
@@ -285,24 +285,24 @@ public sealed class CiscoCsafConnector : VexConnectorBase
|
||||
private sealed record CiscoAdvisoryIndex
|
||||
{
|
||||
public List<CiscoAdvisory>? Advisories { get; init; }
|
||||
public string? Next { get; init; }
|
||||
}
|
||||
|
||||
private sealed record CiscoAdvisory
|
||||
{
|
||||
public string? Id { get; init; }
|
||||
public string? Url { get; init; }
|
||||
public string? Revision { get; init; }
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
public DateTimeOffset? LastModified { get; init; }
|
||||
public string? Sha256 { get; init; }
|
||||
}
|
||||
|
||||
private sealed record CiscoAdvisoryEntry(
|
||||
string Id,
|
||||
Uri DocumentUri,
|
||||
string? Revision,
|
||||
DateTimeOffset? Published,
|
||||
DateTimeOffset? LastModified,
|
||||
string? Sha256);
|
||||
}
|
||||
public string? Next { get; init; }
|
||||
}
|
||||
|
||||
private sealed record CiscoAdvisory
|
||||
{
|
||||
public string? Id { get; init; }
|
||||
public string? Url { get; init; }
|
||||
public string? Revision { get; init; }
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
public DateTimeOffset? LastModified { get; init; }
|
||||
public string? Sha256 { get; init; }
|
||||
}
|
||||
|
||||
private sealed record CiscoAdvisoryEntry(
|
||||
string Id,
|
||||
Uri DocumentUri,
|
||||
string? Revision,
|
||||
DateTimeOffset? Published,
|
||||
DateTimeOffset? LastModified,
|
||||
string? Sha256);
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Postgres\StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
|
||||
|
||||
@@ -16,7 +16,7 @@ using StellaOps.Excititor.Connectors.Abstractions.Trust;
|
||||
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
|
||||
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.MSRC.CSAF;
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Postgres\StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
|
||||
|
||||
@@ -1,266 +1,266 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.Abstractions.Trust;
|
||||
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Oracle.CSAF;
|
||||
|
||||
public sealed class OracleCsafConnector : VexConnectorBase
|
||||
{
|
||||
private static readonly VexConnectorDescriptor DescriptorInstance = new(
|
||||
id: "excititor:oracle",
|
||||
kind: VexProviderKind.Vendor,
|
||||
displayName: "Oracle CSAF")
|
||||
{
|
||||
Tags = ImmutableArray.Create("oracle", "csaf", "cpu"),
|
||||
};
|
||||
|
||||
private readonly OracleCatalogLoader _catalogLoader;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IEnumerable<IVexConnectorOptionsValidator<OracleConnectorOptions>> _validators;
|
||||
|
||||
private OracleConnectorOptions? _options;
|
||||
private OracleCatalogResult? _catalog;
|
||||
|
||||
public OracleCsafConnector(
|
||||
OracleCatalogLoader catalogLoader,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IEnumerable<IVexConnectorOptionsValidator<OracleConnectorOptions>> validators,
|
||||
ILogger<OracleCsafConnector> logger,
|
||||
TimeProvider timeProvider)
|
||||
: base(DescriptorInstance, logger, timeProvider)
|
||||
{
|
||||
_catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader));
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<OracleConnectorOptions>>();
|
||||
}
|
||||
|
||||
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
{
|
||||
_options = VexConnectorOptionsBinder.Bind(
|
||||
Descriptor,
|
||||
settings,
|
||||
validators: _validators);
|
||||
|
||||
_catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
|
||||
LogConnectorEvent(LogLevel.Information, "validate", "Oracle CSAF catalogue loaded.", new Dictionary<string, object?>
|
||||
{
|
||||
["catalogEntryCount"] = _catalog.Metadata.Entries.Length,
|
||||
["scheduleCount"] = _catalog.Metadata.CpuSchedule.Length,
|
||||
["fromOffline"] = _catalog.FromOfflineSnapshot,
|
||||
});
|
||||
}
|
||||
|
||||
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
if (_options is null)
|
||||
{
|
||||
throw new InvalidOperationException("Connector must be validated before fetch operations.");
|
||||
}
|
||||
|
||||
_catalog ??= await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entries = _catalog.Metadata.Entries
|
||||
.OrderBy(static entry => entry.PublishedAt == default ? DateTimeOffset.MinValue : entry.PublishedAt)
|
||||
.ToImmutableArray();
|
||||
|
||||
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
|
||||
var since = ResolveSince(context.Since, state?.LastUpdated);
|
||||
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
|
||||
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
|
||||
var digestList = new List<string>(knownDigests);
|
||||
var latestPublished = state?.LastUpdated ?? since ?? DateTimeOffset.MinValue;
|
||||
var stateChanged = false;
|
||||
|
||||
var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName);
|
||||
|
||||
LogConnectorEvent(LogLevel.Information, "fetch.begin", "Starting Oracle CSAF catalogue iteration.", new Dictionary<string, object?>
|
||||
{
|
||||
["since"] = since?.ToString("O"),
|
||||
["entryCount"] = entries.Length,
|
||||
});
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (ShouldSkipEntry(entry, since))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var expectedDigest = NormalizeDigest(entry.Sha256);
|
||||
if (expectedDigest is not null && digestSet.Contains(expectedDigest))
|
||||
{
|
||||
latestPublished = UpdateLatest(latestPublished, entry.PublishedAt);
|
||||
LogConnectorEvent(LogLevel.Debug, "fetch.skip.cached", "Skipping Oracle CSAF entry because digest already processed.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
["digest"] = expectedDigest,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
var rawDocument = await DownloadEntryAsync(client, entry, cancellationToken).ConfigureAwait(false);
|
||||
if (rawDocument is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (expectedDigest is not null && !string.Equals(rawDocument.Digest, expectedDigest, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Warning, "fetch.checksum_mismatch", "Oracle CSAF document checksum mismatch; document skipped.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
["expected"] = expectedDigest,
|
||||
["actual"] = rawDocument.Digest,
|
||||
["documentUri"] = entry.DocumentUri.ToString(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!digestSet.Add(rawDocument.Digest))
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Debug, "fetch.skip.duplicate", "Oracle CSAF document digest already ingested.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
["digest"] = rawDocument.Digest,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
|
||||
digestList.Add(rawDocument.Digest);
|
||||
stateChanged = true;
|
||||
latestPublished = UpdateLatest(latestPublished, entry.PublishedAt);
|
||||
|
||||
LogConnectorEvent(LogLevel.Information, "fetch.document_ingested", "Oracle CSAF document stored.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
["digest"] = rawDocument.Digest,
|
||||
["documentUri"] = entry.DocumentUri.ToString(),
|
||||
["publishedAt"] = entry.PublishedAt.ToString("O"),
|
||||
});
|
||||
|
||||
yield return rawDocument;
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (stateChanged)
|
||||
{
|
||||
var baseState = state ?? new VexConnectorState(
|
||||
Descriptor.Id,
|
||||
null,
|
||||
ImmutableArray<string>.Empty,
|
||||
ImmutableDictionary<string, string>.Empty,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null);
|
||||
var newState = baseState with
|
||||
{
|
||||
LastUpdated = latestPublished == DateTimeOffset.MinValue ? baseState.LastUpdated : latestPublished,
|
||||
DocumentDigests = digestList.ToImmutableArray(),
|
||||
};
|
||||
|
||||
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var ingestedCount = digestList.Count - knownDigests.Length;
|
||||
LogConnectorEvent(LogLevel.Information, "fetch.complete", "Oracle CSAF fetch completed.", new Dictionary<string, object?>
|
||||
{
|
||||
["stateChanged"] = stateChanged,
|
||||
["documentsProcessed"] = ingestedCount,
|
||||
["latestPublished"] = latestPublished == DateTimeOffset.MinValue ? null : latestPublished.ToString("O"),
|
||||
});
|
||||
}
|
||||
|
||||
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException("OracleCsafConnector relies on dedicated CSAF normalizers.");
|
||||
|
||||
public OracleCatalogResult? GetCachedCatalog() => _catalog;
|
||||
|
||||
private static DateTimeOffset? ResolveSince(DateTimeOffset? contextSince, DateTimeOffset? stateSince)
|
||||
{
|
||||
if (contextSince is null)
|
||||
{
|
||||
return stateSince;
|
||||
}
|
||||
|
||||
if (stateSince is null)
|
||||
{
|
||||
return contextSince;
|
||||
}
|
||||
|
||||
return stateSince > contextSince ? stateSince : contextSince;
|
||||
}
|
||||
|
||||
private static bool ShouldSkipEntry(OracleCatalogEntry entry, DateTimeOffset? since)
|
||||
{
|
||||
if (since is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (entry.PublishedAt == default)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return entry.PublishedAt <= since;
|
||||
}
|
||||
|
||||
private async Task<VexRawDocument?> DownloadEntryAsync(HttpClient client, OracleCatalogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
if (entry.DocumentUri is null)
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Warning, "fetch.skip.missing_uri", "Oracle CSAF entry missing document URI; skipping.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
var payload = await DownloadWithRetryAsync(client, entry.DocumentUri, cancellationToken).ConfigureAwait(false);
|
||||
if (payload is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = BuildMetadata(builder =>
|
||||
{
|
||||
builder.Add("oracle.csaf.entryId", entry.Id);
|
||||
builder.Add("oracle.csaf.title", entry.Title);
|
||||
builder.Add("oracle.csaf.revision", entry.Revision);
|
||||
if (entry.PublishedAt != default)
|
||||
{
|
||||
builder.Add("oracle.csaf.published", entry.PublishedAt.ToString("O"));
|
||||
}
|
||||
|
||||
builder.Add("oracle.csaf.sha256", NormalizeDigest(entry.Sha256));
|
||||
builder.Add("oracle.csaf.size", entry.Size?.ToString(CultureInfo.InvariantCulture));
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Oracle.CSAF;
|
||||
|
||||
public sealed class OracleCsafConnector : VexConnectorBase
|
||||
{
|
||||
private static readonly VexConnectorDescriptor DescriptorInstance = new(
|
||||
id: "excititor:oracle",
|
||||
kind: VexProviderKind.Vendor,
|
||||
displayName: "Oracle CSAF")
|
||||
{
|
||||
Tags = ImmutableArray.Create("oracle", "csaf", "cpu"),
|
||||
};
|
||||
|
||||
private readonly OracleCatalogLoader _catalogLoader;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IVexConnectorStateRepository _stateRepository;
|
||||
private readonly IEnumerable<IVexConnectorOptionsValidator<OracleConnectorOptions>> _validators;
|
||||
|
||||
private OracleConnectorOptions? _options;
|
||||
private OracleCatalogResult? _catalog;
|
||||
|
||||
public OracleCsafConnector(
|
||||
OracleCatalogLoader catalogLoader,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IVexConnectorStateRepository stateRepository,
|
||||
IEnumerable<IVexConnectorOptionsValidator<OracleConnectorOptions>> validators,
|
||||
ILogger<OracleCsafConnector> logger,
|
||||
TimeProvider timeProvider)
|
||||
: base(DescriptorInstance, logger, timeProvider)
|
||||
{
|
||||
_catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader));
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_validators = validators ?? Array.Empty<IVexConnectorOptionsValidator<OracleConnectorOptions>>();
|
||||
}
|
||||
|
||||
public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
{
|
||||
_options = VexConnectorOptionsBinder.Bind(
|
||||
Descriptor,
|
||||
settings,
|
||||
validators: _validators);
|
||||
|
||||
_catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
|
||||
LogConnectorEvent(LogLevel.Information, "validate", "Oracle CSAF catalogue loaded.", new Dictionary<string, object?>
|
||||
{
|
||||
["catalogEntryCount"] = _catalog.Metadata.Entries.Length,
|
||||
["scheduleCount"] = _catalog.Metadata.CpuSchedule.Length,
|
||||
["fromOffline"] = _catalog.FromOfflineSnapshot,
|
||||
});
|
||||
}
|
||||
|
||||
public override async IAsyncEnumerable<VexRawDocument> FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
if (_options is null)
|
||||
{
|
||||
throw new InvalidOperationException("Connector must be validated before fetch operations.");
|
||||
}
|
||||
|
||||
_catalog ??= await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var entries = _catalog.Metadata.Entries
|
||||
.OrderBy(static entry => entry.PublishedAt == default ? DateTimeOffset.MinValue : entry.PublishedAt)
|
||||
.ToImmutableArray();
|
||||
|
||||
var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false);
|
||||
var since = ResolveSince(context.Since, state?.LastUpdated);
|
||||
var knownDigests = state?.DocumentDigests ?? ImmutableArray<string>.Empty;
|
||||
var digestSet = new HashSet<string>(knownDigests, StringComparer.OrdinalIgnoreCase);
|
||||
var digestList = new List<string>(knownDigests);
|
||||
var latestPublished = state?.LastUpdated ?? since ?? DateTimeOffset.MinValue;
|
||||
var stateChanged = false;
|
||||
|
||||
var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName);
|
||||
|
||||
LogConnectorEvent(LogLevel.Information, "fetch.begin", "Starting Oracle CSAF catalogue iteration.", new Dictionary<string, object?>
|
||||
{
|
||||
["since"] = since?.ToString("O"),
|
||||
["entryCount"] = entries.Length,
|
||||
});
|
||||
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (ShouldSkipEntry(entry, since))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var expectedDigest = NormalizeDigest(entry.Sha256);
|
||||
if (expectedDigest is not null && digestSet.Contains(expectedDigest))
|
||||
{
|
||||
latestPublished = UpdateLatest(latestPublished, entry.PublishedAt);
|
||||
LogConnectorEvent(LogLevel.Debug, "fetch.skip.cached", "Skipping Oracle CSAF entry because digest already processed.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
["digest"] = expectedDigest,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
var rawDocument = await DownloadEntryAsync(client, entry, cancellationToken).ConfigureAwait(false);
|
||||
if (rawDocument is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (expectedDigest is not null && !string.Equals(rawDocument.Digest, expectedDigest, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Warning, "fetch.checksum_mismatch", "Oracle CSAF document checksum mismatch; document skipped.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
["expected"] = expectedDigest,
|
||||
["actual"] = rawDocument.Digest,
|
||||
["documentUri"] = entry.DocumentUri.ToString(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!digestSet.Add(rawDocument.Digest))
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Debug, "fetch.skip.duplicate", "Oracle CSAF document digest already ingested.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
["digest"] = rawDocument.Digest,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false);
|
||||
digestList.Add(rawDocument.Digest);
|
||||
stateChanged = true;
|
||||
latestPublished = UpdateLatest(latestPublished, entry.PublishedAt);
|
||||
|
||||
LogConnectorEvent(LogLevel.Information, "fetch.document_ingested", "Oracle CSAF document stored.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
["digest"] = rawDocument.Digest,
|
||||
["documentUri"] = entry.DocumentUri.ToString(),
|
||||
["publishedAt"] = entry.PublishedAt.ToString("O"),
|
||||
});
|
||||
|
||||
yield return rawDocument;
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (stateChanged)
|
||||
{
|
||||
var baseState = state ?? new VexConnectorState(
|
||||
Descriptor.Id,
|
||||
null,
|
||||
ImmutableArray<string>.Empty,
|
||||
ImmutableDictionary<string, string>.Empty,
|
||||
null,
|
||||
0,
|
||||
null,
|
||||
null);
|
||||
var newState = baseState with
|
||||
{
|
||||
LastUpdated = latestPublished == DateTimeOffset.MinValue ? baseState.LastUpdated : latestPublished,
|
||||
DocumentDigests = digestList.ToImmutableArray(),
|
||||
};
|
||||
|
||||
await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var ingestedCount = digestList.Count - knownDigests.Length;
|
||||
LogConnectorEvent(LogLevel.Information, "fetch.complete", "Oracle CSAF fetch completed.", new Dictionary<string, object?>
|
||||
{
|
||||
["stateChanged"] = stateChanged,
|
||||
["documentsProcessed"] = ingestedCount,
|
||||
["latestPublished"] = latestPublished == DateTimeOffset.MinValue ? null : latestPublished.ToString("O"),
|
||||
});
|
||||
}
|
||||
|
||||
public override ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException("OracleCsafConnector relies on dedicated CSAF normalizers.");
|
||||
|
||||
public OracleCatalogResult? GetCachedCatalog() => _catalog;
|
||||
|
||||
private static DateTimeOffset? ResolveSince(DateTimeOffset? contextSince, DateTimeOffset? stateSince)
|
||||
{
|
||||
if (contextSince is null)
|
||||
{
|
||||
return stateSince;
|
||||
}
|
||||
|
||||
if (stateSince is null)
|
||||
{
|
||||
return contextSince;
|
||||
}
|
||||
|
||||
return stateSince > contextSince ? stateSince : contextSince;
|
||||
}
|
||||
|
||||
private static bool ShouldSkipEntry(OracleCatalogEntry entry, DateTimeOffset? since)
|
||||
{
|
||||
if (since is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (entry.PublishedAt == default)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return entry.PublishedAt <= since;
|
||||
}
|
||||
|
||||
private async Task<VexRawDocument?> DownloadEntryAsync(HttpClient client, OracleCatalogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
if (entry.DocumentUri is null)
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Warning, "fetch.skip.missing_uri", "Oracle CSAF entry missing document URI; skipping.", new Dictionary<string, object?>
|
||||
{
|
||||
["entryId"] = entry.Id,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
var payload = await DownloadWithRetryAsync(client, entry.DocumentUri, cancellationToken).ConfigureAwait(false);
|
||||
if (payload is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var metadata = BuildMetadata(builder =>
|
||||
{
|
||||
builder.Add("oracle.csaf.entryId", entry.Id);
|
||||
builder.Add("oracle.csaf.title", entry.Title);
|
||||
builder.Add("oracle.csaf.revision", entry.Revision);
|
||||
if (entry.PublishedAt != default)
|
||||
{
|
||||
builder.Add("oracle.csaf.published", entry.PublishedAt.ToString("O"));
|
||||
}
|
||||
|
||||
builder.Add("oracle.csaf.sha256", NormalizeDigest(entry.Sha256));
|
||||
builder.Add("oracle.csaf.size", entry.Size?.ToString(CultureInfo.InvariantCulture));
|
||||
if (!entry.Products.IsDefaultOrEmpty)
|
||||
{
|
||||
builder.Add("oracle.csaf.products", string.Join(",", entry.Products));
|
||||
@@ -268,96 +268,96 @@ public sealed class OracleCsafConnector : VexConnectorBase
|
||||
|
||||
ConnectorSignerMetadataEnricher.Enrich(builder, Descriptor.Id, _logger);
|
||||
});
|
||||
|
||||
return CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload.AsMemory(), metadata);
|
||||
}
|
||||
|
||||
private async Task<byte[]?> DownloadWithRetryAsync(HttpClient client, Uri uri, CancellationToken cancellationToken)
|
||||
{
|
||||
const int maxAttempts = 3;
|
||||
var delay = TimeSpan.FromSeconds(1);
|
||||
|
||||
for (var attempt = 1; attempt <= maxAttempts; attempt++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
using var response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
if (IsTransient(response.StatusCode) && attempt < maxAttempts)
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Warning, "fetch.retry.status", "Oracle CSAF document request returned transient status; retrying.", new Dictionary<string, object?>
|
||||
{
|
||||
["status"] = (int)response.StatusCode,
|
||||
["attempt"] = attempt,
|
||||
["uri"] = uri.ToString(),
|
||||
});
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
delay = delay + delay;
|
||||
continue;
|
||||
}
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
var bytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
return bytes;
|
||||
}
|
||||
catch (Exception ex) when (IsTransient(ex) && attempt < maxAttempts)
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Warning, "fetch.retry.exception", "Oracle CSAF document request failed; retrying.", new Dictionary<string, object?>
|
||||
{
|
||||
["attempt"] = attempt,
|
||||
["uri"] = uri.ToString(),
|
||||
["exception"] = ex.GetType().Name,
|
||||
});
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
delay = delay + delay;
|
||||
}
|
||||
}
|
||||
|
||||
LogConnectorEvent(LogLevel.Error, "fetch.failed", "Oracle CSAF document could not be retrieved after retries.", new Dictionary<string, object?>
|
||||
{
|
||||
["uri"] = uri.ToString(),
|
||||
});
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsTransient(Exception exception)
|
||||
=> exception is HttpRequestException or IOException or TaskCanceledException;
|
||||
|
||||
private static bool IsTransient(HttpStatusCode statusCode)
|
||||
{
|
||||
var status = (int)statusCode;
|
||||
return status is >= 500 or 408 or 429;
|
||||
}
|
||||
|
||||
private static string? NormalizeDigest(string? digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = digest.Trim();
|
||||
if (!trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
trimmed = "sha256:" + trimmed;
|
||||
}
|
||||
|
||||
return trimmed.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static DateTimeOffset UpdateLatest(DateTimeOffset current, DateTimeOffset published)
|
||||
{
|
||||
if (published == default)
|
||||
{
|
||||
return current;
|
||||
}
|
||||
|
||||
return published > current ? published : current;
|
||||
}
|
||||
}
|
||||
|
||||
return CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload.AsMemory(), metadata);
|
||||
}
|
||||
|
||||
private async Task<byte[]?> DownloadWithRetryAsync(HttpClient client, Uri uri, CancellationToken cancellationToken)
|
||||
{
|
||||
const int maxAttempts = 3;
|
||||
var delay = TimeSpan.FromSeconds(1);
|
||||
|
||||
for (var attempt = 1; attempt <= maxAttempts; attempt++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
using var response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
if (IsTransient(response.StatusCode) && attempt < maxAttempts)
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Warning, "fetch.retry.status", "Oracle CSAF document request returned transient status; retrying.", new Dictionary<string, object?>
|
||||
{
|
||||
["status"] = (int)response.StatusCode,
|
||||
["attempt"] = attempt,
|
||||
["uri"] = uri.ToString(),
|
||||
});
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
delay = delay + delay;
|
||||
continue;
|
||||
}
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
var bytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
return bytes;
|
||||
}
|
||||
catch (Exception ex) when (IsTransient(ex) && attempt < maxAttempts)
|
||||
{
|
||||
LogConnectorEvent(LogLevel.Warning, "fetch.retry.exception", "Oracle CSAF document request failed; retrying.", new Dictionary<string, object?>
|
||||
{
|
||||
["attempt"] = attempt,
|
||||
["uri"] = uri.ToString(),
|
||||
["exception"] = ex.GetType().Name,
|
||||
});
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
delay = delay + delay;
|
||||
}
|
||||
}
|
||||
|
||||
LogConnectorEvent(LogLevel.Error, "fetch.failed", "Oracle CSAF document could not be retrieved after retries.", new Dictionary<string, object?>
|
||||
{
|
||||
["uri"] = uri.ToString(),
|
||||
});
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsTransient(Exception exception)
|
||||
=> exception is HttpRequestException or IOException or TaskCanceledException;
|
||||
|
||||
private static bool IsTransient(HttpStatusCode statusCode)
|
||||
{
|
||||
var status = (int)statusCode;
|
||||
return status is >= 500 or 408 or 429;
|
||||
}
|
||||
|
||||
private static string? NormalizeDigest(string? digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = digest.Trim();
|
||||
if (!trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
trimmed = "sha256:" + trimmed;
|
||||
}
|
||||
|
||||
return trimmed.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static DateTimeOffset UpdateLatest(DateTimeOffset current, DateTimeOffset published)
|
||||
{
|
||||
if (published == default)
|
||||
{
|
||||
return current;
|
||||
}
|
||||
|
||||
return published > current ? published : current;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Postgres\StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
|
||||
|
||||
@@ -5,7 +5,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using System.IO.Abstractions;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
|
||||
|
||||
@@ -11,7 +11,7 @@ using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.RedHat.CSAF;
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Postgres\StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
|
||||
|
||||
@@ -20,7 +20,7 @@ using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
|
||||
using StellaOps.Excititor.Connectors.Abstractions.Trust;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub;
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
|
||||
|
||||
public sealed record RancherHubCheckpointState(
|
||||
@@ -15,84 +15,84 @@ public sealed record RancherHubCheckpointState(
|
||||
ImmutableArray<string> Digests);
|
||||
|
||||
public sealed class RancherHubCheckpointManager
|
||||
{
|
||||
private const string CheckpointPrefix = "checkpoint:";
|
||||
private readonly IVexConnectorStateRepository _repository;
|
||||
|
||||
public RancherHubCheckpointManager(IVexConnectorStateRepository repository)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
}
|
||||
|
||||
public async ValueTask<RancherHubCheckpointState> LoadAsync(string connectorId, VexConnectorContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var state = await _repository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false);
|
||||
var cursor = ExtractCursor(state?.DocumentDigests ?? ImmutableArray<string>.Empty);
|
||||
var digests = ExtractDigests(state?.DocumentDigests ?? ImmutableArray<string>.Empty);
|
||||
var lastPublishedAt = state?.LastUpdated;
|
||||
var effectiveSince = context.Since;
|
||||
|
||||
if (context.Settings.Values.TryGetValue("checkpoint", out var checkpointOverride) && !string.IsNullOrWhiteSpace(checkpointOverride))
|
||||
{
|
||||
cursor = checkpointOverride;
|
||||
digests = ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
if (effectiveSince is null && lastPublishedAt is not null)
|
||||
{
|
||||
effectiveSince = lastPublishedAt;
|
||||
}
|
||||
|
||||
if (effectiveSince is not null && lastPublishedAt is not null && effectiveSince < lastPublishedAt)
|
||||
{
|
||||
digests = ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
return new RancherHubCheckpointState(cursor, lastPublishedAt, effectiveSince, digests);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(string connectorId, string? cursor, DateTimeOffset? lastPublishedAt, ImmutableArray<string> digests, CancellationToken cancellationToken)
|
||||
{
|
||||
var entries = ImmutableArray.CreateBuilder<string>();
|
||||
if (!string.IsNullOrWhiteSpace(cursor))
|
||||
{
|
||||
entries.Add($"{CheckpointPrefix}{cursor}");
|
||||
}
|
||||
|
||||
foreach (var digest in digests)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (digest.StartsWith(CheckpointPrefix, StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.Add(digest);
|
||||
}
|
||||
|
||||
var state = new VexConnectorState(connectorId, lastPublishedAt, entries.ToImmutable());
|
||||
return _repository.SaveAsync(state, cancellationToken);
|
||||
}
|
||||
|
||||
private static string? ExtractCursor(ImmutableArray<string> digests)
|
||||
{
|
||||
foreach (var entry in digests)
|
||||
{
|
||||
if (entry.StartsWith(CheckpointPrefix, StringComparison.Ordinal))
|
||||
{
|
||||
return entry[CheckpointPrefix.Length..];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractDigests(ImmutableArray<string> digests)
|
||||
=> digests.Where(d => !d.StartsWith(CheckpointPrefix, StringComparison.Ordinal)).ToImmutableArray();
|
||||
}
|
||||
{
|
||||
private const string CheckpointPrefix = "checkpoint:";
|
||||
private readonly IVexConnectorStateRepository _repository;
|
||||
|
||||
public RancherHubCheckpointManager(IVexConnectorStateRepository repository)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
}
|
||||
|
||||
public async ValueTask<RancherHubCheckpointState> LoadAsync(string connectorId, VexConnectorContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var state = await _repository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false);
|
||||
var cursor = ExtractCursor(state?.DocumentDigests ?? ImmutableArray<string>.Empty);
|
||||
var digests = ExtractDigests(state?.DocumentDigests ?? ImmutableArray<string>.Empty);
|
||||
var lastPublishedAt = state?.LastUpdated;
|
||||
var effectiveSince = context.Since;
|
||||
|
||||
if (context.Settings.Values.TryGetValue("checkpoint", out var checkpointOverride) && !string.IsNullOrWhiteSpace(checkpointOverride))
|
||||
{
|
||||
cursor = checkpointOverride;
|
||||
digests = ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
if (effectiveSince is null && lastPublishedAt is not null)
|
||||
{
|
||||
effectiveSince = lastPublishedAt;
|
||||
}
|
||||
|
||||
if (effectiveSince is not null && lastPublishedAt is not null && effectiveSince < lastPublishedAt)
|
||||
{
|
||||
digests = ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
return new RancherHubCheckpointState(cursor, lastPublishedAt, effectiveSince, digests);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(string connectorId, string? cursor, DateTimeOffset? lastPublishedAt, ImmutableArray<string> digests, CancellationToken cancellationToken)
|
||||
{
|
||||
var entries = ImmutableArray.CreateBuilder<string>();
|
||||
if (!string.IsNullOrWhiteSpace(cursor))
|
||||
{
|
||||
entries.Add($"{CheckpointPrefix}{cursor}");
|
||||
}
|
||||
|
||||
foreach (var digest in digests)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (digest.StartsWith(CheckpointPrefix, StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.Add(digest);
|
||||
}
|
||||
|
||||
var state = new VexConnectorState(connectorId, lastPublishedAt, entries.ToImmutable());
|
||||
return _repository.SaveAsync(state, cancellationToken);
|
||||
}
|
||||
|
||||
private static string? ExtractCursor(ImmutableArray<string> digests)
|
||||
{
|
||||
foreach (var entry in digests)
|
||||
{
|
||||
if (entry.StartsWith(CheckpointPrefix, StringComparison.Ordinal))
|
||||
{
|
||||
return entry[CheckpointPrefix.Length..];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> ExtractDigests(ImmutableArray<string> digests)
|
||||
=> digests.Where(d => !d.StartsWith(CheckpointPrefix, StringComparison.Ordinal)).ToImmutableArray();
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Postgres\StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Postgres\StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.0-preview.7.25380.108" />
|
||||
|
||||
@@ -14,7 +14,7 @@ using StellaOps.Excititor.Connectors.Abstractions.Trust;
|
||||
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF;
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ Provide ingestion/domain logic for VEX observations and linksets under the Aggre
|
||||
|
||||
## Roles
|
||||
- Backend library engineer (.NET 10 / C# preview).
|
||||
- QA automation (unit + integration against Mongo fixtures).
|
||||
- QA automation (unit + integration against Postgres or in-memory fixtures).
|
||||
|
||||
## Working Agreements
|
||||
1. Update sprint status on task transitions; log notable decisions in sprint Execution Log.
|
||||
@@ -28,7 +28,7 @@ Provide ingestion/domain logic for VEX observations and linksets under the Aggre
|
||||
## Testing & Determinism
|
||||
- Write deterministic tests: seeded clocks/GUIDs, stable ordering of collections, ISO-8601 UTC timestamps.
|
||||
- Cover linkset extraction ordering, supersede chain construction, and duplicate prevention.
|
||||
- Use Mongo in-memory/test harness fixtures; do not rely on live services.
|
||||
- Use Postgres test fixtures or in-memory harnesses; do not rely on live services.
|
||||
|
||||
## Boundaries
|
||||
- Do not embed Policy Engine rules or Cartographer schemas here; expose contracts for consumers instead.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Persistent state snapshot for a connector run (resume tokens, failure counts, checkpoints).
|
||||
/// </summary>
|
||||
public sealed record VexConnectorState(
|
||||
string ConnectorId,
|
||||
DateTimeOffset? LastUpdated,
|
||||
ImmutableArray<string> DocumentDigests,
|
||||
ImmutableDictionary<string, string> ResumeTokens = default,
|
||||
DateTimeOffset? LastSuccessAt = null,
|
||||
int FailureCount = 0,
|
||||
DateTimeOffset? NextEligibleRun = null,
|
||||
string? LastFailureReason = null,
|
||||
DateTimeOffset? LastCheckpoint = null)
|
||||
{
|
||||
public ImmutableDictionary<string, string> ResumeTokens { get; init; } = ResumeTokens.IsDefault
|
||||
? ImmutableDictionary<string, string>.Empty
|
||||
: ResumeTokens;
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Repository abstraction for connector state persistence.
|
||||
/// </summary>
|
||||
public interface IVexConnectorStateRepository
|
||||
{
|
||||
ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provider registry persistence abstraction.
|
||||
/// </summary>
|
||||
public interface IVexProviderStore
|
||||
{
|
||||
ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Claim store abstraction for VEX statements.
|
||||
/// </summary>
|
||||
public interface IVexClaimStore
|
||||
{
|
||||
ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,710 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory provider store used while Postgres implementations are brought online.
|
||||
/// </summary>
|
||||
public sealed class InMemoryVexProviderStore : IVexProviderStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, VexProvider> _providers = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
_providers.TryGetValue(id, out var provider);
|
||||
return ValueTask.FromResult<VexProvider?>(provider);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(provider);
|
||||
_providers[provider.Id] = provider;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexProvider>>(_providers.Values.ToList());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory connector state repository for deterministic tests and temporary storage.
|
||||
/// </summary>
|
||||
public sealed class InMemoryVexConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, VexConnectorState> _states = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
{
|
||||
_states.TryGetValue(connectorId, out var state);
|
||||
return ValueTask.FromResult<VexConnectorState?>(state);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
_states[state.ConnectorId] = state with { LastUpdated = state.LastUpdated ?? DateTimeOffset.UtcNow };
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexConnectorState>>(_states.Values.ToList());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory claim store used while Mongo dependencies are removed.
|
||||
/// </summary>
|
||||
public sealed class InMemoryVexClaimStore : IVexClaimStore
|
||||
{
|
||||
private readonly ConcurrentBag<VexClaim> _claims = new();
|
||||
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(claims);
|
||||
foreach (var claim in claims)
|
||||
{
|
||||
_claims.Add(claim);
|
||||
}
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = _claims.Where(c =>
|
||||
string.Equals(c.VulnerabilityId, vulnerabilityId, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(c.Product.Key, productKey, StringComparison.OrdinalIgnoreCase) &&
|
||||
(!since.HasValue || c.LastSeen >= since.Value))
|
||||
.ToList();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(results);
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = _claims
|
||||
.Where(c => string.Equals(c.VulnerabilityId, vulnerabilityId, StringComparison.OrdinalIgnoreCase))
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(results);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory raw document store used for tests and sealed-mode fixtures while Mongo is removed.
|
||||
/// Implements the same semantics as the Postgres raw store: canonical JSON, deterministic digests,
|
||||
/// tenant scoping, and stable ordering.
|
||||
/// </summary>
|
||||
public sealed class InMemoryVexRawStore : IVexRawStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, VexRawRecord> _records = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly int _inlineThreshold;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public InMemoryVexRawStore(int inlineThresholdBytes = 256 * 1024, TimeProvider? timeProvider = null)
|
||||
{
|
||||
_inlineThreshold = Math.Max(1, inlineThresholdBytes);
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var canonicalContent = CanonicalizeJson(document.Content);
|
||||
var digest = EnsureDigest(document.Digest, canonicalContent);
|
||||
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
|
||||
var tenant = ResolveTenant(metadata);
|
||||
var format = document.Format;
|
||||
var retrievedAt = document.RetrievedAt;
|
||||
var inline = canonicalContent.Length <= _inlineThreshold;
|
||||
var recordedAt = _timeProvider.GetUtcNow();
|
||||
|
||||
var record = new VexRawRecord(
|
||||
digest,
|
||||
tenant,
|
||||
document.ProviderId,
|
||||
format,
|
||||
document.SourceUri,
|
||||
retrievedAt,
|
||||
metadata,
|
||||
inline ? canonicalContent : canonicalContent.ToArray(),
|
||||
inline,
|
||||
metadata.TryGetValue("supersedes", out var supersedes) ? supersedes : null,
|
||||
metadata.TryGetValue("etag", out var etag) ? etag : null,
|
||||
recordedAt);
|
||||
|
||||
_records.AddOrUpdate(digest, record, (_, existing) => existing);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<VexRawRecord?> FindByDigestAsync(string digest, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
_records.TryGetValue(digest, out var record);
|
||||
return ValueTask.FromResult<VexRawRecord?>(record);
|
||||
}
|
||||
|
||||
public ValueTask<VexRawDocumentPage> QueryAsync(VexRawQuery query, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(query);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var filtered = _records.Values
|
||||
.Where(r => string.Equals(r.Tenant, query.Tenant, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(r => query.ProviderIds.Count == 0 || query.ProviderIds.Contains(r.ProviderId, StringComparer.OrdinalIgnoreCase))
|
||||
.Where(r => query.Digests.Count == 0 || query.Digests.Contains(r.Digest, StringComparer.OrdinalIgnoreCase))
|
||||
.Where(r => query.Formats.Count == 0 || query.Formats.Contains(r.Format))
|
||||
.Where(r => query.Since is null || r.RetrievedAt >= query.Since.Value)
|
||||
.Where(r => query.Until is null || r.RetrievedAt <= query.Until.Value)
|
||||
.OrderByDescending(r => r.RetrievedAt)
|
||||
.ThenByDescending(r => r.Digest, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
if (query.Cursor is not null)
|
||||
{
|
||||
filtered = filtered
|
||||
.Where(r =>
|
||||
r.RetrievedAt < query.Cursor.RetrievedAt ||
|
||||
(r.RetrievedAt == query.Cursor.RetrievedAt && string.CompareOrdinal(r.Digest, query.Cursor.Digest) < 0))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
var page = filtered.Take(query.Limit).ToList();
|
||||
var hasMore = filtered.Count > page.Count;
|
||||
var nextCursor = hasMore && page.Count > 0
|
||||
? new VexRawCursor(page[^1].RetrievedAt, page[^1].Digest)
|
||||
: null;
|
||||
|
||||
var summaries = page
|
||||
.Select(r => new VexRawDocumentSummary(
|
||||
r.Digest,
|
||||
r.ProviderId,
|
||||
r.Format,
|
||||
r.SourceUri,
|
||||
r.RetrievedAt,
|
||||
r.InlineContent,
|
||||
r.Metadata))
|
||||
.ToList();
|
||||
|
||||
return ValueTask.FromResult(new VexRawDocumentPage(summaries, nextCursor, hasMore));
|
||||
}
|
||||
|
||||
private static string ResolveTenant(IReadOnlyDictionary<string, string> metadata)
|
||||
{
|
||||
if (metadata.TryGetValue("tenant", out var tenant) && !string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
return tenant.Trim();
|
||||
}
|
||||
|
||||
return "default";
|
||||
}
|
||||
|
||||
private static byte[] CanonicalizeJson(ReadOnlyMemory<byte> content)
|
||||
{
|
||||
using var jsonDocument = JsonDocument.Parse(content);
|
||||
using var buffer = new ArrayBufferWriter<byte>();
|
||||
using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions { Indented = false }))
|
||||
{
|
||||
WriteCanonical(writer, jsonDocument.RootElement);
|
||||
}
|
||||
|
||||
return buffer.WrittenMemory.ToArray();
|
||||
}
|
||||
|
||||
private static void WriteCanonical(Utf8JsonWriter writer, JsonElement element)
|
||||
{
|
||||
switch (element.ValueKind)
|
||||
{
|
||||
case JsonValueKind.Object:
|
||||
writer.WriteStartObject();
|
||||
foreach (var property in element.EnumerateObject().OrderBy(p => p.Name, StringComparer.Ordinal))
|
||||
{
|
||||
writer.WritePropertyName(property.Name);
|
||||
WriteCanonical(writer, property.Value);
|
||||
}
|
||||
writer.WriteEndObject();
|
||||
break;
|
||||
case JsonValueKind.Array:
|
||||
writer.WriteStartArray();
|
||||
foreach (var item in element.EnumerateArray())
|
||||
{
|
||||
WriteCanonical(writer, item);
|
||||
}
|
||||
writer.WriteEndArray();
|
||||
break;
|
||||
case JsonValueKind.String:
|
||||
writer.WriteStringValue(element.GetString());
|
||||
break;
|
||||
case JsonValueKind.Number:
|
||||
if (element.TryGetInt64(out var l))
|
||||
{
|
||||
writer.WriteNumberValue(l);
|
||||
}
|
||||
else if (element.TryGetDouble(out var d))
|
||||
{
|
||||
writer.WriteNumberValue(d);
|
||||
}
|
||||
else
|
||||
{
|
||||
writer.WriteRawValue(element.GetRawText());
|
||||
}
|
||||
|
||||
break;
|
||||
case JsonValueKind.True:
|
||||
writer.WriteBooleanValue(true);
|
||||
break;
|
||||
case JsonValueKind.False:
|
||||
writer.WriteBooleanValue(false);
|
||||
break;
|
||||
case JsonValueKind.Null:
|
||||
case JsonValueKind.Undefined:
|
||||
writer.WriteNullValue();
|
||||
break;
|
||||
default:
|
||||
writer.WriteRawValue(element.GetRawText());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private static string EnsureDigest(string digest, ReadOnlyMemory<byte> canonicalContent)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(digest) && digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return digest;
|
||||
}
|
||||
|
||||
Span<byte> hash = stackalloc byte[32];
|
||||
if (!System.Security.Cryptography.SHA256.TryHashData(canonicalContent.Span, hash, out _))
|
||||
{
|
||||
hash = System.Security.Cryptography.SHA256.HashData(canonicalContent.ToArray());
|
||||
}
|
||||
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory append-only linkset store implementing both append semantics and read models.
|
||||
/// </summary>
|
||||
public sealed class InMemoryAppendOnlyLinksetStore : IAppendOnlyLinksetStore, IVexLinksetStore
|
||||
{
|
||||
private readonly Dictionary<string, VexLinkset> _linksets = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, List<LinksetMutationEvent>> _mutations = new(StringComparer.OrdinalIgnoreCase);
|
||||
private long _sequenceNumber;
|
||||
private readonly object _lock = new();
|
||||
|
||||
public ValueTask<AppendLinksetResult> AppendObservationAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
VexLinksetObservationRefModel observation,
|
||||
VexProductScope scope,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
return AppendObservationsBatchAsync(tenant, vulnerabilityId, productKey, new[] { observation }, scope, cancellationToken);
|
||||
}
|
||||
|
||||
public ValueTask<AppendLinksetResult> AppendObservationsBatchAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
IEnumerable<VexLinksetObservationRefModel> observations,
|
||||
VexProductScope scope,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
var linksetId = VexLinkset.CreateLinksetId(tenant, vulnerabilityId, productKey);
|
||||
var key = CreateKey(tenant, linksetId);
|
||||
var wasCreated = false;
|
||||
if (!_linksets.TryGetValue(key, out var linkset))
|
||||
{
|
||||
wasCreated = true;
|
||||
linkset = new VexLinkset(
|
||||
linksetId,
|
||||
tenant,
|
||||
vulnerabilityId,
|
||||
productKey,
|
||||
scope,
|
||||
Enumerable.Empty<VexLinksetObservationRefModel>(),
|
||||
null,
|
||||
DateTimeOffset.UtcNow,
|
||||
DateTimeOffset.UtcNow);
|
||||
_linksets[key] = linkset;
|
||||
AddMutation(key, LinksetMutationEvent.MutationTypes.LinksetCreated, null, null, null, null);
|
||||
}
|
||||
|
||||
var existingObsIds = new HashSet<string>(linkset.Observations.Select(o => o.ObservationId), StringComparer.Ordinal);
|
||||
var newObservations = observations
|
||||
.Where(o => o is not null && !existingObsIds.Contains(o.ObservationId))
|
||||
.ToList();
|
||||
|
||||
var observationsAdded = 0;
|
||||
if (newObservations.Count > 0)
|
||||
{
|
||||
observationsAdded = newObservations.Count;
|
||||
var merged = linkset.Observations.Concat(newObservations);
|
||||
linkset = linkset.WithObservations(merged, linkset.Disagreements);
|
||||
_linksets[key] = linkset;
|
||||
|
||||
foreach (var obs in newObservations)
|
||||
{
|
||||
AddMutation(key, LinksetMutationEvent.MutationTypes.ObservationAdded, obs.ObservationId, obs.ProviderId, obs.Status, obs.Confidence);
|
||||
}
|
||||
}
|
||||
|
||||
var sequence = _sequenceNumber;
|
||||
return ValueTask.FromResult(wasCreated
|
||||
? AppendLinksetResult.Created(linkset, observationsAdded, sequence)
|
||||
: observationsAdded > 0
|
||||
? AppendLinksetResult.Updated(linkset, observationsAdded, 0, sequence)
|
||||
: AppendLinksetResult.NoChange(linkset, sequence));
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<AppendLinksetResult> AppendDisagreementAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
VexObservationDisagreement disagreement,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
var linksetId = VexLinkset.CreateLinksetId(tenant, vulnerabilityId, productKey);
|
||||
var key = CreateKey(tenant, linksetId);
|
||||
var wasCreated = false;
|
||||
if (!_linksets.TryGetValue(key, out var linkset))
|
||||
{
|
||||
wasCreated = true;
|
||||
linkset = new VexLinkset(
|
||||
linksetId,
|
||||
tenant,
|
||||
vulnerabilityId,
|
||||
productKey,
|
||||
new VexProductScope(productKey, null, null, productKey, null, Array.Empty<string>()),
|
||||
Enumerable.Empty<VexLinksetObservationRefModel>(),
|
||||
Enumerable.Empty<VexObservationDisagreement>(),
|
||||
DateTimeOffset.UtcNow,
|
||||
DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
var disagreements = linkset.Disagreements.ToList();
|
||||
var existing = disagreements.Any(d =>
|
||||
string.Equals(d.ProviderId, disagreement.ProviderId, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(d.Status, disagreement.Status, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(d.Justification, disagreement.Justification, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
var disagreementsAdded = 0;
|
||||
if (!existing)
|
||||
{
|
||||
disagreements.Add(disagreement);
|
||||
disagreementsAdded = 1;
|
||||
}
|
||||
|
||||
var updated = linkset.WithObservations(linkset.Observations, disagreements);
|
||||
_linksets[key] = updated;
|
||||
|
||||
if (wasCreated)
|
||||
{
|
||||
AddMutation(key, LinksetMutationEvent.MutationTypes.LinksetCreated, null, null, null, null);
|
||||
}
|
||||
|
||||
if (disagreementsAdded > 0)
|
||||
{
|
||||
AddMutation(key, LinksetMutationEvent.MutationTypes.DisagreementAdded, null, disagreement.ProviderId, disagreement.Status, disagreement.Confidence);
|
||||
}
|
||||
|
||||
var sequence = _sequenceNumber;
|
||||
return ValueTask.FromResult(disagreementsAdded > 0 || wasCreated
|
||||
? AppendLinksetResult.Updated(updated, 0, disagreementsAdded, sequence)
|
||||
: AppendLinksetResult.NoChange(updated, sequence));
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<VexLinkset?> GetByIdAsync(string tenant, string linksetId, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(GetByKeyInternal(tenant, linksetId));
|
||||
|
||||
public ValueTask<VexLinkset?> GetByKeyAsync(string tenant, string vulnerabilityId, string productKey, CancellationToken cancellationToken)
|
||||
{
|
||||
var linksetId = VexLinkset.CreateLinksetId(tenant, vulnerabilityId, productKey);
|
||||
return ValueTask.FromResult(GetByKeyInternal(tenant, linksetId));
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<VexLinkset>> FindByVulnerabilityAsync(string tenant, string vulnerabilityId, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var results = _linksets.Values
|
||||
.Where(ls => string.Equals(ls.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(ls => string.Equals(ls.VulnerabilityId, vulnerabilityId, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderByDescending(ls => ls.UpdatedAt)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
return ValueTask.FromResult<IReadOnlyList<VexLinkset>>(results);
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<VexLinkset>> FindByProductKeyAsync(string tenant, string productKey, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var results = _linksets.Values
|
||||
.Where(ls => string.Equals(ls.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(ls => string.Equals(ls.ProductKey, productKey, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderByDescending(ls => ls.UpdatedAt)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
return ValueTask.FromResult<IReadOnlyList<VexLinkset>>(results);
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<VexLinkset>> FindWithConflictsAsync(string tenant, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var results = _linksets.Values
|
||||
.Where(ls => string.Equals(ls.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(ls => ls.HasConflicts)
|
||||
.OrderByDescending(ls => ls.UpdatedAt)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
return ValueTask.FromResult<IReadOnlyList<VexLinkset>>(results);
|
||||
}
|
||||
|
||||
public ValueTask<long> CountAsync(string tenant, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var count = _linksets.Values.Count(ls => string.Equals(ls.Tenant, tenant, StringComparison.OrdinalIgnoreCase));
|
||||
return ValueTask.FromResult((long)count);
|
||||
}
|
||||
|
||||
public ValueTask<long> CountWithConflictsAsync(string tenant, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var count = _linksets.Values.Count(ls =>
|
||||
string.Equals(ls.Tenant, tenant, StringComparison.OrdinalIgnoreCase) && ls.HasConflicts);
|
||||
return ValueTask.FromResult((long)count);
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<LinksetMutationEvent>> GetMutationLogAsync(string tenant, string linksetId, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = CreateKey(tenant, linksetId);
|
||||
if (_mutations.TryGetValue(key, out var log))
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<LinksetMutationEvent>>(log.ToList());
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<LinksetMutationEvent>>(Array.Empty<LinksetMutationEvent>());
|
||||
}
|
||||
|
||||
public ValueTask<bool> InsertAsync(VexLinkset linkset, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (_lock)
|
||||
{
|
||||
var key = CreateKey(linkset.Tenant, linkset.LinksetId);
|
||||
if (_linksets.ContainsKey(key))
|
||||
{
|
||||
return ValueTask.FromResult(false);
|
||||
}
|
||||
|
||||
_linksets[key] = linkset;
|
||||
AddMutation(key, LinksetMutationEvent.MutationTypes.LinksetCreated, null, null, null, null);
|
||||
return ValueTask.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<bool> UpsertAsync(VexLinkset linkset, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (_lock)
|
||||
{
|
||||
var key = CreateKey(linkset.Tenant, linkset.LinksetId);
|
||||
var created = !_linksets.ContainsKey(key);
|
||||
_linksets[key] = linkset;
|
||||
if (created)
|
||||
{
|
||||
AddMutation(key, LinksetMutationEvent.MutationTypes.LinksetCreated, null, null, null, null);
|
||||
}
|
||||
|
||||
return ValueTask.FromResult(created);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<VexLinkset> GetOrCreateAsync(string tenant, string vulnerabilityId, string productKey, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (_lock)
|
||||
{
|
||||
var linksetId = VexLinkset.CreateLinksetId(tenant, vulnerabilityId, productKey);
|
||||
var key = CreateKey(tenant, linksetId);
|
||||
if (_linksets.TryGetValue(key, out var existing))
|
||||
{
|
||||
return ValueTask.FromResult(existing);
|
||||
}
|
||||
|
||||
var scope = new VexProductScope(productKey, null, null, productKey, null, Array.Empty<string>());
|
||||
var linkset = new VexLinkset(linksetId, tenant, vulnerabilityId, productKey, scope, Enumerable.Empty<VexLinksetObservationRefModel>());
|
||||
_linksets[key] = linkset;
|
||||
AddMutation(key, LinksetMutationEvent.MutationTypes.LinksetCreated, null, null, null, null);
|
||||
return ValueTask.FromResult(linkset);
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<VexLinkset>> FindByProviderAsync(string tenant, string providerId, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var results = _linksets.Values
|
||||
.Where(ls => string.Equals(ls.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
|
||||
.Where(ls => ls.Observations.Any(o => string.Equals(o.ProviderId, providerId, StringComparison.OrdinalIgnoreCase)))
|
||||
.OrderByDescending(ls => ls.UpdatedAt)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<VexLinkset>>(results);
|
||||
}
|
||||
|
||||
public ValueTask<bool> DeleteAsync(string tenant, string linksetId, CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
lock (_lock)
|
||||
{
|
||||
var key = CreateKey(tenant, linksetId);
|
||||
var removed = _linksets.Remove(key);
|
||||
_mutations.Remove(key);
|
||||
return ValueTask.FromResult(removed);
|
||||
}
|
||||
}
|
||||
|
||||
private VexLinkset? GetByKeyInternal(string tenant, string linksetId)
|
||||
{
|
||||
var key = CreateKey(tenant, linksetId);
|
||||
_linksets.TryGetValue(key, out var linkset);
|
||||
return linkset;
|
||||
}
|
||||
|
||||
private void AddMutation(string key, string mutationType, string? observationId, string? providerId, string? status, double? confidence)
|
||||
{
|
||||
var sequence = ++_sequenceNumber;
|
||||
if (!_mutations.TryGetValue(key, out var log))
|
||||
{
|
||||
log = new List<LinksetMutationEvent>();
|
||||
_mutations[key] = log;
|
||||
}
|
||||
|
||||
log.Add(new LinksetMutationEvent(sequence, mutationType, DateTimeOffset.UtcNow, observationId, providerId, status, confidence, null));
|
||||
}
|
||||
|
||||
private static string CreateKey(string tenant, string linksetId)
|
||||
=> $"{tenant.Trim().ToLowerInvariant()}|{linksetId}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory observation store to unblock APIs while Postgres backing store is implemented.
|
||||
/// </summary>
|
||||
public sealed class InMemoryVexObservationStore : IVexObservationStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, VexObservation>> _tenants = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public ValueTask<bool> InsertAsync(VexObservation observation, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(observation);
|
||||
var tenantStore = _tenants.GetOrAdd(observation.Tenant, _ => new ConcurrentDictionary<string, VexObservation>(StringComparer.OrdinalIgnoreCase));
|
||||
var inserted = tenantStore.TryAdd(observation.ObservationId, observation);
|
||||
return ValueTask.FromResult(inserted);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> UpsertAsync(VexObservation observation, CancellationToken cancellationToken)
|
||||
{
|
||||
await InsertAsync(observation, cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
|
||||
public ValueTask<int> InsertManyAsync(string tenant, IEnumerable<VexObservation> observations, CancellationToken cancellationToken)
|
||||
{
|
||||
if (observations is null)
|
||||
{
|
||||
return ValueTask.FromResult(0);
|
||||
}
|
||||
|
||||
var count = 0;
|
||||
foreach (var obs in observations)
|
||||
{
|
||||
if (string.Equals(obs.Tenant, tenant, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (InsertAsync(obs, cancellationToken).Result)
|
||||
{
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.FromResult(count);
|
||||
}
|
||||
|
||||
public ValueTask<VexObservation?> GetByIdAsync(string tenant, string observationId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tenants.TryGetValue(tenant, out var store) && store.TryGetValue(observationId, out var observation))
|
||||
{
|
||||
return ValueTask.FromResult<VexObservation?>(observation);
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<VexObservation?>(null);
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<VexObservation>> FindByVulnerabilityAndProductAsync(string tenant, string vulnerabilityId, string productKey, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = _tenants.TryGetValue(tenant, out var store)
|
||||
? store.Values
|
||||
.Where(o => o.Statements.Any(s =>
|
||||
string.Equals(s.VulnerabilityId, vulnerabilityId, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(s.ProductKey, productKey, StringComparison.OrdinalIgnoreCase)))
|
||||
.OrderByDescending(o => o.CreatedAt)
|
||||
.ToList()
|
||||
: new List<VexObservation>();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<VexObservation>>(results);
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyList<VexObservation>> FindByProviderAsync(string tenant, string providerId, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = _tenants.TryGetValue(tenant, out var store)
|
||||
? store.Values
|
||||
.Where(o => string.Equals(o.ProviderId, providerId, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderByDescending(o => o.CreatedAt)
|
||||
.Take(limit)
|
||||
.ToList()
|
||||
: new List<VexObservation>();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<VexObservation>>(results);
|
||||
}
|
||||
|
||||
public ValueTask<bool> DeleteAsync(string tenant, string observationId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tenants.TryGetValue(tenant, out var store))
|
||||
{
|
||||
return ValueTask.FromResult(store.TryRemove(observationId, out _));
|
||||
}
|
||||
|
||||
return ValueTask.FromResult(false);
|
||||
}
|
||||
|
||||
public ValueTask<long> CountAsync(string tenant, CancellationToken cancellationToken)
|
||||
{
|
||||
var count = _tenants.TryGetValue(tenant, out var store)
|
||||
? store.Count
|
||||
: 0;
|
||||
return ValueTask.FromResult((long)count);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
// Temporary stubs to allow legacy interfaces to compile while MongoDB is removed.
|
||||
// These types are intentionally minimal; they do not perform any database operations.
|
||||
namespace MongoDB.Driver;
|
||||
|
||||
public interface IClientSessionHandle : IAsyncDisposable, IDisposable
|
||||
{
|
||||
}
|
||||
@@ -8,7 +8,7 @@ using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Export;
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Policy\StellaOps.Excititor.Policy.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Storage.Postgres\StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
|
||||
namespace StellaOps.Excititor.Export;
|
||||
|
||||
|
||||
@@ -1,112 +1,110 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.Cisco.CSAF;
|
||||
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO.Abstractions.TestingHelpers;
|
||||
using Xunit;
|
||||
using System.Threading;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors;
|
||||
|
||||
public sealed class CiscoCsafConnectorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task FetchAsync_NewAdvisory_StoresDocumentAndUpdatesState()
|
||||
{
|
||||
var responses = new Dictionary<Uri, Queue<HttpResponseMessage>>
|
||||
{
|
||||
[new Uri("https://api.cisco.test/.well-known/csaf/provider-metadata.json")] = QueueResponses("""
|
||||
{
|
||||
"metadata": {
|
||||
"publisher": {
|
||||
"name": "Cisco",
|
||||
"category": "vendor",
|
||||
"contact_details": { "id": "excititor:cisco" }
|
||||
}
|
||||
},
|
||||
"distributions": {
|
||||
"directories": [ "https://api.cisco.test/csaf/" ]
|
||||
}
|
||||
}
|
||||
"""),
|
||||
[new Uri("https://api.cisco.test/csaf/index.json")] = QueueResponses("""
|
||||
{
|
||||
"advisories": [
|
||||
{
|
||||
"id": "cisco-sa-2025",
|
||||
"url": "https://api.cisco.test/csaf/cisco-sa-2025.json",
|
||||
"published": "2025-10-01T00:00:00Z",
|
||||
"lastModified": "2025-10-02T00:00:00Z",
|
||||
"sha256": "cafebabe"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""),
|
||||
[new Uri("https://api.cisco.test/csaf/cisco-sa-2025.json")] = QueueResponses("{ \"document\": \"payload\" }")
|
||||
};
|
||||
|
||||
var handler = new RoutingHttpMessageHandler(responses);
|
||||
var httpClient = new HttpClient(handler);
|
||||
var factory = new SingleHttpClientFactory(httpClient);
|
||||
var metadataLoader = new CiscoProviderMetadataLoader(
|
||||
factory,
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
Options.Create(new CiscoConnectorOptions
|
||||
{
|
||||
MetadataUri = "https://api.cisco.test/.well-known/csaf/provider-metadata.json",
|
||||
PersistOfflineSnapshot = false,
|
||||
}),
|
||||
NullLogger<CiscoProviderMetadataLoader>.Instance,
|
||||
new MockFileSystem());
|
||||
|
||||
var stateRepository = new InMemoryConnectorStateRepository();
|
||||
var connector = new CiscoCsafConnector(
|
||||
metadataLoader,
|
||||
factory,
|
||||
stateRepository,
|
||||
new[] { new CiscoConnectorOptionsValidator() },
|
||||
NullLogger<CiscoCsafConnector>.Instance,
|
||||
TimeProvider.System);
|
||||
|
||||
var settings = new VexConnectorSettings(ImmutableDictionary<string, string>.Empty);
|
||||
await connector.ValidateAsync(settings, CancellationToken.None);
|
||||
|
||||
var sink = new InMemoryRawSink();
|
||||
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider(), ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
var documents = new List<VexRawDocument>();
|
||||
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
|
||||
{
|
||||
documents.Add(doc);
|
||||
}
|
||||
|
||||
documents.Should().HaveCount(1);
|
||||
sink.Documents.Should().HaveCount(1);
|
||||
stateRepository.CurrentState.Should().NotBeNull();
|
||||
stateRepository.CurrentState!.DocumentDigests.Should().HaveCount(1);
|
||||
|
||||
// second run should not refetch documents
|
||||
sink.Documents.Clear();
|
||||
documents.Clear();
|
||||
|
||||
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
|
||||
{
|
||||
documents.Add(doc);
|
||||
}
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.Cisco.CSAF;
|
||||
using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO.Abstractions.TestingHelpers;
|
||||
using Xunit;
|
||||
using System.Threading;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors;
|
||||
|
||||
public sealed class CiscoCsafConnectorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task FetchAsync_NewAdvisory_StoresDocumentAndUpdatesState()
|
||||
{
|
||||
var responses = new Dictionary<Uri, Queue<HttpResponseMessage>>
|
||||
{
|
||||
[new Uri("https://api.cisco.test/.well-known/csaf/provider-metadata.json")] = QueueResponses("""
|
||||
{
|
||||
"metadata": {
|
||||
"publisher": {
|
||||
"name": "Cisco",
|
||||
"category": "vendor",
|
||||
"contact_details": { "id": "excititor:cisco" }
|
||||
}
|
||||
},
|
||||
"distributions": {
|
||||
"directories": [ "https://api.cisco.test/csaf/" ]
|
||||
}
|
||||
}
|
||||
"""),
|
||||
[new Uri("https://api.cisco.test/csaf/index.json")] = QueueResponses("""
|
||||
{
|
||||
"advisories": [
|
||||
{
|
||||
"id": "cisco-sa-2025",
|
||||
"url": "https://api.cisco.test/csaf/cisco-sa-2025.json",
|
||||
"published": "2025-10-01T00:00:00Z",
|
||||
"lastModified": "2025-10-02T00:00:00Z",
|
||||
"sha256": "cafebabe"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""),
|
||||
[new Uri("https://api.cisco.test/csaf/cisco-sa-2025.json")] = QueueResponses("{ \"document\": \"payload\" }")
|
||||
};
|
||||
|
||||
var handler = new RoutingHttpMessageHandler(responses);
|
||||
var httpClient = new HttpClient(handler);
|
||||
var factory = new SingleHttpClientFactory(httpClient);
|
||||
var metadataLoader = new CiscoProviderMetadataLoader(
|
||||
factory,
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
Options.Create(new CiscoConnectorOptions
|
||||
{
|
||||
MetadataUri = "https://api.cisco.test/.well-known/csaf/provider-metadata.json",
|
||||
PersistOfflineSnapshot = false,
|
||||
}),
|
||||
NullLogger<CiscoProviderMetadataLoader>.Instance,
|
||||
new MockFileSystem());
|
||||
|
||||
var stateRepository = new InMemoryConnectorStateRepository();
|
||||
var connector = new CiscoCsafConnector(
|
||||
metadataLoader,
|
||||
factory,
|
||||
stateRepository,
|
||||
new[] { new CiscoConnectorOptionsValidator() },
|
||||
NullLogger<CiscoCsafConnector>.Instance,
|
||||
TimeProvider.System);
|
||||
|
||||
var settings = new VexConnectorSettings(ImmutableDictionary<string, string>.Empty);
|
||||
await connector.ValidateAsync(settings, CancellationToken.None);
|
||||
|
||||
var sink = new InMemoryRawSink();
|
||||
var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider(), ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
var documents = new List<VexRawDocument>();
|
||||
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
|
||||
{
|
||||
documents.Add(doc);
|
||||
}
|
||||
|
||||
documents.Should().HaveCount(1);
|
||||
sink.Documents.Should().HaveCount(1);
|
||||
stateRepository.CurrentState.Should().NotBeNull();
|
||||
stateRepository.CurrentState!.DocumentDigests.Should().HaveCount(1);
|
||||
|
||||
// second run should not refetch documents
|
||||
sink.Documents.Clear();
|
||||
documents.Clear();
|
||||
|
||||
await foreach (var doc in connector.FetchAsync(context, CancellationToken.None))
|
||||
{
|
||||
documents.Add(doc);
|
||||
}
|
||||
|
||||
documents.Should().BeEmpty();
|
||||
sink.Documents.Should().BeEmpty();
|
||||
}
|
||||
@@ -225,60 +223,60 @@ public sealed class CiscoCsafConnectorTests
|
||||
savedProvider.Trust.Cosign.IdentityPattern.Should().Be("https://sig.example.com/*");
|
||||
savedProvider.Trust.PgpFingerprints.Should().Contain(new[] { "0123456789ABCDEF", "FEDCBA9876543210" });
|
||||
}
|
||||
|
||||
private static Queue<HttpResponseMessage> QueueResponses(string payload)
|
||||
=> new(new[]
|
||||
{
|
||||
new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(payload, Encoding.UTF8, "application/json"),
|
||||
}
|
||||
});
|
||||
|
||||
private sealed class RoutingHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Dictionary<Uri, Queue<HttpResponseMessage>> _responses;
|
||||
|
||||
public RoutingHttpMessageHandler(Dictionary<Uri, Queue<HttpResponseMessage>> responses)
|
||||
{
|
||||
_responses = responses;
|
||||
}
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var queue) && queue.Count > 0)
|
||||
{
|
||||
var response = queue.Peek();
|
||||
return Task.FromResult(response.Clone());
|
||||
}
|
||||
|
||||
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
|
||||
{
|
||||
Content = new StringContent($"No response configured for {request.RequestUri}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SingleHttpClientFactory : IHttpClientFactory
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public SingleHttpClientFactory(HttpClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public HttpClient CreateClient(string name) => _client;
|
||||
}
|
||||
|
||||
|
||||
private static Queue<HttpResponseMessage> QueueResponses(string payload)
|
||||
=> new(new[]
|
||||
{
|
||||
new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(payload, Encoding.UTF8, "application/json"),
|
||||
}
|
||||
});
|
||||
|
||||
private sealed class RoutingHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Dictionary<Uri, Queue<HttpResponseMessage>> _responses;
|
||||
|
||||
public RoutingHttpMessageHandler(Dictionary<Uri, Queue<HttpResponseMessage>> responses)
|
||||
{
|
||||
_responses = responses;
|
||||
}
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var queue) && queue.Count > 0)
|
||||
{
|
||||
var response = queue.Peek();
|
||||
return Task.FromResult(response.Clone());
|
||||
}
|
||||
|
||||
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
|
||||
{
|
||||
Content = new StringContent($"No response configured for {request.RequestUri}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SingleHttpClientFactory : IHttpClientFactory
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public SingleHttpClientFactory(HttpClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public HttpClient CreateClient(string name) => _client;
|
||||
}
|
||||
|
||||
private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
public VexConnectorState? CurrentState { get; private set; }
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(CurrentState);
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
CurrentState = state;
|
||||
return ValueTask.CompletedTask;
|
||||
@@ -289,59 +287,59 @@ public sealed class CiscoCsafConnectorTests
|
||||
{
|
||||
public List<VexProvider> SavedProviders { get; } = new();
|
||||
|
||||
public ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexProvider?>(null);
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexProvider>>(Array.Empty<VexProvider>());
|
||||
|
||||
public ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken)
|
||||
{
|
||||
SavedProviders.Add(provider);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryRawSink : IVexRawDocumentSink
|
||||
{
|
||||
public List<VexRawDocument> Documents { get; } = new();
|
||||
|
||||
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
Documents.Add(document);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
internal static class HttpResponseMessageExtensions
|
||||
{
|
||||
public static HttpResponseMessage Clone(this HttpResponseMessage response)
|
||||
{
|
||||
var clone = new HttpResponseMessage(response.StatusCode);
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
clone.Headers.TryAddWithoutValidation(header.Key, header.Value);
|
||||
}
|
||||
|
||||
if (response.Content is not null)
|
||||
{
|
||||
var payload = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
|
||||
clone.Content = new StringContent(payload, Encoding.UTF8, response.Content.Headers.ContentType?.MediaType);
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryRawSink : IVexRawDocumentSink
|
||||
{
|
||||
public List<VexRawDocument> Documents { get; } = new();
|
||||
|
||||
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
Documents.Add(document);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
internal static class HttpResponseMessageExtensions
|
||||
{
|
||||
public static HttpResponseMessage Clone(this HttpResponseMessage response)
|
||||
{
|
||||
var clone = new HttpResponseMessage(response.StatusCode);
|
||||
foreach (var header in response.Headers)
|
||||
{
|
||||
clone.Headers.TryAddWithoutValidation(header.Key, header.Value);
|
||||
}
|
||||
|
||||
if (response.Content is not null)
|
||||
{
|
||||
var payload = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
|
||||
clone.Content = new StringContent(payload, Encoding.UTF8, response.Content.Headers.ContentType?.MediaType);
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,9 +16,7 @@ using StellaOps.Excititor.Connectors.MSRC.CSAF;
|
||||
using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication;
|
||||
using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using Xunit;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Connectors;
|
||||
|
||||
@@ -323,10 +321,10 @@ public sealed class MsrcCsafConnectorTests
|
||||
{
|
||||
public VexConnectorState? State { get; private set; }
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(State);
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
State = state;
|
||||
return ValueTask.CompletedTask;
|
||||
|
||||
@@ -17,10 +17,8 @@ using StellaOps.Excititor.Connectors.Oracle.CSAF;
|
||||
using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using System.IO.Abstractions.TestingHelpers;
|
||||
using Xunit;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Connectors;
|
||||
|
||||
@@ -257,10 +255,10 @@ public sealed class OracleCsafConnectorTests
|
||||
{
|
||||
public VexConnectorState? State { get; private set; }
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(State);
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
State = state;
|
||||
return ValueTask.CompletedTask;
|
||||
|
||||
@@ -1,78 +1,76 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.Connectors;
|
||||
|
||||
public sealed class RedHatCsafConnectorTests
|
||||
{
|
||||
private static readonly VexConnectorDescriptor Descriptor = new("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF");
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_EmitsDocumentsAfterSince()
|
||||
{
|
||||
var metadata = """
|
||||
{
|
||||
"metadata": {
|
||||
"provider": { "name": "Red Hat Product Security" }
|
||||
},
|
||||
"distributions": [
|
||||
{ "directory": "https://example.com/security/data/csaf/v2/advisories/" }
|
||||
],
|
||||
"rolie": {
|
||||
"feeds": [
|
||||
{ "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" }
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
var feed = """
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<entry>
|
||||
<id>urn:redhat:1</id>
|
||||
<updated>2025-10-16T10:00:00Z</updated>
|
||||
<link href="https://example.com/doc1.json" rel="enclosure" />
|
||||
</entry>
|
||||
<entry>
|
||||
<id>urn:redhat:2</id>
|
||||
<updated>2025-10-17T10:00:00Z</updated>
|
||||
<link href="https://example.com/doc2.json" rel="enclosure" />
|
||||
</entry>
|
||||
</feed>
|
||||
""";
|
||||
|
||||
var handler = TestHttpMessageHandler.Create(
|
||||
request => Response(HttpStatusCode.OK, metadata, "application/json"),
|
||||
request => Response(HttpStatusCode.OK, feed, "application/atom+xml"),
|
||||
request => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json"));
|
||||
|
||||
var httpClient = new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://example.com/"),
|
||||
};
|
||||
|
||||
var factory = new SingleClientHttpClientFactory(httpClient);
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var options = Options.Create(new RedHatConnectorOptions());
|
||||
var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger<RedHatProviderMetadataLoader>.Instance);
|
||||
var stateRepository = new InMemoryConnectorStateRepository();
|
||||
var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger<RedHatCsafConnector>.Instance, TimeProvider.System);
|
||||
|
||||
var rawSink = new CapturingRawSink();
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.Connectors;
|
||||
|
||||
public sealed class RedHatCsafConnectorTests
|
||||
{
|
||||
private static readonly VexConnectorDescriptor Descriptor = new("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF");
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_EmitsDocumentsAfterSince()
|
||||
{
|
||||
var metadata = """
|
||||
{
|
||||
"metadata": {
|
||||
"provider": { "name": "Red Hat Product Security" }
|
||||
},
|
||||
"distributions": [
|
||||
{ "directory": "https://example.com/security/data/csaf/v2/advisories/" }
|
||||
],
|
||||
"rolie": {
|
||||
"feeds": [
|
||||
{ "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" }
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
var feed = """
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<entry>
|
||||
<id>urn:redhat:1</id>
|
||||
<updated>2025-10-16T10:00:00Z</updated>
|
||||
<link href="https://example.com/doc1.json" rel="enclosure" />
|
||||
</entry>
|
||||
<entry>
|
||||
<id>urn:redhat:2</id>
|
||||
<updated>2025-10-17T10:00:00Z</updated>
|
||||
<link href="https://example.com/doc2.json" rel="enclosure" />
|
||||
</entry>
|
||||
</feed>
|
||||
""";
|
||||
|
||||
var handler = TestHttpMessageHandler.Create(
|
||||
request => Response(HttpStatusCode.OK, metadata, "application/json"),
|
||||
request => Response(HttpStatusCode.OK, feed, "application/atom+xml"),
|
||||
request => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json"));
|
||||
|
||||
var httpClient = new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://example.com/"),
|
||||
};
|
||||
|
||||
var factory = new SingleClientHttpClientFactory(httpClient);
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var options = Options.Create(new RedHatConnectorOptions());
|
||||
var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger<RedHatProviderMetadataLoader>.Instance);
|
||||
var stateRepository = new InMemoryConnectorStateRepository();
|
||||
var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger<RedHatCsafConnector>.Instance, TimeProvider.System);
|
||||
|
||||
var rawSink = new CapturingRawSink();
|
||||
var context = new VexConnectorContext(
|
||||
new DateTimeOffset(2025, 10, 16, 12, 0, 0, TimeSpan.Zero),
|
||||
VexConnectorSettings.Empty,
|
||||
@@ -81,164 +79,164 @@ public sealed class RedHatCsafConnectorTests
|
||||
new NoopNormalizerRouter(),
|
||||
new ServiceCollection().BuildServiceProvider(),
|
||||
ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
var results = new List<VexRawDocument>();
|
||||
await foreach (var document in connector.FetchAsync(context, CancellationToken.None))
|
||||
{
|
||||
results.Add(document);
|
||||
}
|
||||
|
||||
Assert.Single(results);
|
||||
Assert.Single(rawSink.Documents);
|
||||
Assert.Equal("https://example.com/doc2.json", results[0].SourceUri.ToString());
|
||||
Assert.Equal("https://example.com/doc2.json", rawSink.Documents[0].SourceUri.ToString());
|
||||
Assert.Equal(3, handler.CallCount);
|
||||
stateRepository.State.Should().NotBeNull();
|
||||
stateRepository.State!.LastUpdated.Should().Be(new DateTimeOffset(2025, 10, 17, 10, 0, 0, TimeSpan.Zero));
|
||||
stateRepository.State.DocumentDigests.Should().HaveCount(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_UsesStateToSkipDuplicateDocuments()
|
||||
{
|
||||
var metadata = """
|
||||
{
|
||||
"metadata": {
|
||||
"provider": { "name": "Red Hat Product Security" }
|
||||
},
|
||||
"distributions": [
|
||||
{ "directory": "https://example.com/security/data/csaf/v2/advisories/" }
|
||||
],
|
||||
"rolie": {
|
||||
"feeds": [
|
||||
{ "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" }
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
var feed = """
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<entry>
|
||||
<id>urn:redhat:1</id>
|
||||
<updated>2025-10-17T10:00:00Z</updated>
|
||||
<link href="https://example.com/doc1.json" rel="enclosure" />
|
||||
</entry>
|
||||
</feed>
|
||||
""";
|
||||
|
||||
var handler1 = TestHttpMessageHandler.Create(
|
||||
_ => Response(HttpStatusCode.OK, metadata, "application/json"),
|
||||
_ => Response(HttpStatusCode.OK, feed, "application/atom+xml"),
|
||||
_ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json"));
|
||||
|
||||
var stateRepository = new InMemoryConnectorStateRepository();
|
||||
await ExecuteFetchAsync(handler1, stateRepository);
|
||||
|
||||
stateRepository.State.Should().NotBeNull();
|
||||
var previousState = stateRepository.State!;
|
||||
|
||||
var handler2 = TestHttpMessageHandler.Create(
|
||||
_ => Response(HttpStatusCode.OK, metadata, "application/json"),
|
||||
_ => Response(HttpStatusCode.OK, feed, "application/atom+xml"),
|
||||
_ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json"));
|
||||
|
||||
var (results, rawSink) = await ExecuteFetchAsync(handler2, stateRepository);
|
||||
|
||||
results.Should().BeEmpty();
|
||||
rawSink.Documents.Should().BeEmpty();
|
||||
stateRepository.State!.DocumentDigests.Should().Equal(previousState.DocumentDigests);
|
||||
}
|
||||
|
||||
private static HttpResponseMessage Response(HttpStatusCode statusCode, string content, string contentType)
|
||||
=> new(statusCode)
|
||||
{
|
||||
Content = new StringContent(content, Encoding.UTF8, contentType),
|
||||
};
|
||||
|
||||
private sealed class CapturingRawSink : IVexRawDocumentSink
|
||||
{
|
||||
public List<VexRawDocument> Documents { get; } = new();
|
||||
|
||||
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
Documents.Add(document);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
|
||||
private sealed class SingleClientHttpClientFactory : IHttpClientFactory
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public SingleClientHttpClientFactory(HttpClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public HttpClient CreateClient(string name) => _client;
|
||||
}
|
||||
|
||||
private sealed class TestHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Queue<Func<HttpRequestMessage, HttpResponseMessage>> _responders;
|
||||
|
||||
private TestHttpMessageHandler(IEnumerable<Func<HttpRequestMessage, HttpResponseMessage>> responders)
|
||||
{
|
||||
_responders = new Queue<Func<HttpRequestMessage, HttpResponseMessage>>(responders);
|
||||
}
|
||||
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public static TestHttpMessageHandler Create(params Func<HttpRequestMessage, HttpResponseMessage>[] responders)
|
||||
=> new(responders);
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
CallCount++;
|
||||
if (_responders.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("No responder configured for request.");
|
||||
}
|
||||
|
||||
var responder = _responders.Count > 1
|
||||
? _responders.Dequeue()
|
||||
: _responders.Peek();
|
||||
|
||||
var response = responder(request);
|
||||
response.RequestMessage = request;
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<(List<VexRawDocument> Documents, CapturingRawSink Sink)> ExecuteFetchAsync(
|
||||
TestHttpMessageHandler handler,
|
||||
InMemoryConnectorStateRepository stateRepository)
|
||||
{
|
||||
var httpClient = new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://example.com/"),
|
||||
};
|
||||
|
||||
var factory = new SingleClientHttpClientFactory(httpClient);
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var options = Options.Create(new RedHatConnectorOptions());
|
||||
var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger<RedHatProviderMetadataLoader>.Instance);
|
||||
var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger<RedHatCsafConnector>.Instance, TimeProvider.System);
|
||||
|
||||
var rawSink = new CapturingRawSink();
|
||||
|
||||
var results = new List<VexRawDocument>();
|
||||
await foreach (var document in connector.FetchAsync(context, CancellationToken.None))
|
||||
{
|
||||
results.Add(document);
|
||||
}
|
||||
|
||||
Assert.Single(results);
|
||||
Assert.Single(rawSink.Documents);
|
||||
Assert.Equal("https://example.com/doc2.json", results[0].SourceUri.ToString());
|
||||
Assert.Equal("https://example.com/doc2.json", rawSink.Documents[0].SourceUri.ToString());
|
||||
Assert.Equal(3, handler.CallCount);
|
||||
stateRepository.State.Should().NotBeNull();
|
||||
stateRepository.State!.LastUpdated.Should().Be(new DateTimeOffset(2025, 10, 17, 10, 0, 0, TimeSpan.Zero));
|
||||
stateRepository.State.DocumentDigests.Should().HaveCount(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_UsesStateToSkipDuplicateDocuments()
|
||||
{
|
||||
var metadata = """
|
||||
{
|
||||
"metadata": {
|
||||
"provider": { "name": "Red Hat Product Security" }
|
||||
},
|
||||
"distributions": [
|
||||
{ "directory": "https://example.com/security/data/csaf/v2/advisories/" }
|
||||
],
|
||||
"rolie": {
|
||||
"feeds": [
|
||||
{ "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" }
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
var feed = """
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<entry>
|
||||
<id>urn:redhat:1</id>
|
||||
<updated>2025-10-17T10:00:00Z</updated>
|
||||
<link href="https://example.com/doc1.json" rel="enclosure" />
|
||||
</entry>
|
||||
</feed>
|
||||
""";
|
||||
|
||||
var handler1 = TestHttpMessageHandler.Create(
|
||||
_ => Response(HttpStatusCode.OK, metadata, "application/json"),
|
||||
_ => Response(HttpStatusCode.OK, feed, "application/atom+xml"),
|
||||
_ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json"));
|
||||
|
||||
var stateRepository = new InMemoryConnectorStateRepository();
|
||||
await ExecuteFetchAsync(handler1, stateRepository);
|
||||
|
||||
stateRepository.State.Should().NotBeNull();
|
||||
var previousState = stateRepository.State!;
|
||||
|
||||
var handler2 = TestHttpMessageHandler.Create(
|
||||
_ => Response(HttpStatusCode.OK, metadata, "application/json"),
|
||||
_ => Response(HttpStatusCode.OK, feed, "application/atom+xml"),
|
||||
_ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json"));
|
||||
|
||||
var (results, rawSink) = await ExecuteFetchAsync(handler2, stateRepository);
|
||||
|
||||
results.Should().BeEmpty();
|
||||
rawSink.Documents.Should().BeEmpty();
|
||||
stateRepository.State!.DocumentDigests.Should().Equal(previousState.DocumentDigests);
|
||||
}
|
||||
|
||||
private static HttpResponseMessage Response(HttpStatusCode statusCode, string content, string contentType)
|
||||
=> new(statusCode)
|
||||
{
|
||||
Content = new StringContent(content, Encoding.UTF8, contentType),
|
||||
};
|
||||
|
||||
private sealed class CapturingRawSink : IVexRawDocumentSink
|
||||
{
|
||||
public List<VexRawDocument> Documents { get; } = new();
|
||||
|
||||
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
Documents.Add(document);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
|
||||
private sealed class SingleClientHttpClientFactory : IHttpClientFactory
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public SingleClientHttpClientFactory(HttpClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public HttpClient CreateClient(string name) => _client;
|
||||
}
|
||||
|
||||
private sealed class TestHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Queue<Func<HttpRequestMessage, HttpResponseMessage>> _responders;
|
||||
|
||||
private TestHttpMessageHandler(IEnumerable<Func<HttpRequestMessage, HttpResponseMessage>> responders)
|
||||
{
|
||||
_responders = new Queue<Func<HttpRequestMessage, HttpResponseMessage>>(responders);
|
||||
}
|
||||
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public static TestHttpMessageHandler Create(params Func<HttpRequestMessage, HttpResponseMessage>[] responders)
|
||||
=> new(responders);
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
CallCount++;
|
||||
if (_responders.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("No responder configured for request.");
|
||||
}
|
||||
|
||||
var responder = _responders.Count > 1
|
||||
? _responders.Dequeue()
|
||||
: _responders.Peek();
|
||||
|
||||
var response = responder(request);
|
||||
response.RequestMessage = request;
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<(List<VexRawDocument> Documents, CapturingRawSink Sink)> ExecuteFetchAsync(
|
||||
TestHttpMessageHandler handler,
|
||||
InMemoryConnectorStateRepository stateRepository)
|
||||
{
|
||||
var httpClient = new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://example.com/"),
|
||||
};
|
||||
|
||||
var factory = new SingleClientHttpClientFactory(httpClient);
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var options = Options.Create(new RedHatConnectorOptions());
|
||||
var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger<RedHatProviderMetadataLoader>.Instance);
|
||||
var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger<RedHatCsafConnector>.Instance, TimeProvider.System);
|
||||
|
||||
var rawSink = new CapturingRawSink();
|
||||
var context = new VexConnectorContext(
|
||||
null,
|
||||
VexConnectorSettings.Empty,
|
||||
@@ -247,21 +245,21 @@ public sealed class RedHatCsafConnectorTests
|
||||
new NoopNormalizerRouter(),
|
||||
new ServiceCollection().BuildServiceProvider(),
|
||||
ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
var documents = new List<VexRawDocument>();
|
||||
await foreach (var document in connector.FetchAsync(context, CancellationToken.None))
|
||||
{
|
||||
documents.Add(document);
|
||||
}
|
||||
|
||||
return (documents, rawSink);
|
||||
}
|
||||
|
||||
private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
public VexConnectorState? State { get; private set; }
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
|
||||
var documents = new List<VexRawDocument>();
|
||||
await foreach (var document in connector.FetchAsync(context, CancellationToken.None))
|
||||
{
|
||||
documents.Add(document);
|
||||
}
|
||||
|
||||
return (documents, rawSink);
|
||||
}
|
||||
|
||||
private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
public VexConnectorState? State { get; private set; }
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
@@ -271,10 +269,10 @@ public sealed class RedHatCsafConnectorTests
|
||||
return ValueTask.FromResult<VexConnectorState?>(null);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
State = state;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,10 +9,10 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="20.0.28" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
||||
@@ -1,35 +1,34 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests.Connectors;
|
||||
|
||||
public sealed class RancherHubConnectorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task FetchAsync_OfflineSnapshot_StoresDocumentAndUpdatesCheckpoint()
|
||||
{
|
||||
using var fixture = await ConnectorFixture.CreateAsync();
|
||||
|
||||
var sink = new InMemoryRawSink();
|
||||
var context = fixture.CreateContext(sink);
|
||||
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata;
|
||||
using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State;
|
||||
using StellaOps.Excititor.Core;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests.Connectors;
|
||||
|
||||
public sealed class RancherHubConnectorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task FetchAsync_OfflineSnapshot_StoresDocumentAndUpdatesCheckpoint()
|
||||
{
|
||||
using var fixture = await ConnectorFixture.CreateAsync();
|
||||
|
||||
var sink = new InMemoryRawSink();
|
||||
var context = fixture.CreateContext(sink);
|
||||
|
||||
var documents = await CollectAsync(fixture.Connector.FetchAsync(context, CancellationToken.None));
|
||||
|
||||
documents.Should().HaveCount(1);
|
||||
@@ -49,28 +48,28 @@ public sealed class RancherHubConnectorTests
|
||||
"vex.provenance.pgp.fingerprints",
|
||||
"11223344556677889900AABBCCDDEEFF00112233,AABBCCDDEEFF00112233445566778899AABBCCDD");
|
||||
sink.Documents.Should().HaveCount(1);
|
||||
|
||||
var state = fixture.StateRepository.State;
|
||||
state.Should().NotBeNull();
|
||||
state!.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-19T12:00:00Z", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal));
|
||||
state.DocumentDigests.Should().Contain(fixture.ExpectedDocumentDigest);
|
||||
state.DocumentDigests.Should().Contain("checkpoint:cursor-2");
|
||||
state.DocumentDigests.Count.Should().BeLessOrEqualTo(ConnectorFixture.MaxDigestHistory + 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_WhenDocumentDownloadFails_QuarantinesEvent()
|
||||
{
|
||||
using var fixture = await ConnectorFixture.CreateAsync();
|
||||
|
||||
fixture.Handler.SetRoute(fixture.DocumentUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError));
|
||||
|
||||
var sink = new InMemoryRawSink();
|
||||
var context = fixture.CreateContext(sink);
|
||||
|
||||
var documents = await CollectAsync(fixture.Connector.FetchAsync(context, CancellationToken.None));
|
||||
|
||||
documents.Should().BeEmpty();
|
||||
|
||||
var state = fixture.StateRepository.State;
|
||||
state.Should().NotBeNull();
|
||||
state!.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-19T12:00:00Z", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal));
|
||||
state.DocumentDigests.Should().Contain(fixture.ExpectedDocumentDigest);
|
||||
state.DocumentDigests.Should().Contain("checkpoint:cursor-2");
|
||||
state.DocumentDigests.Count.Should().BeLessOrEqualTo(ConnectorFixture.MaxDigestHistory + 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_WhenDocumentDownloadFails_QuarantinesEvent()
|
||||
{
|
||||
using var fixture = await ConnectorFixture.CreateAsync();
|
||||
|
||||
fixture.Handler.SetRoute(fixture.DocumentUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError));
|
||||
|
||||
var sink = new InMemoryRawSink();
|
||||
var context = fixture.CreateContext(sink);
|
||||
|
||||
var documents = await CollectAsync(fixture.Connector.FetchAsync(context, CancellationToken.None));
|
||||
|
||||
documents.Should().BeEmpty();
|
||||
sink.Documents.Should().HaveCount(1);
|
||||
var quarantined = sink.Documents[0];
|
||||
quarantined.Metadata.Should().Contain("rancher.event.quarantine", "true");
|
||||
@@ -80,205 +79,205 @@ public sealed class RancherHubConnectorTests
|
||||
quarantined.Metadata.Should().Contain("vex.provenance.trust.tier", "hub");
|
||||
|
||||
var state = fixture.StateRepository.State;
|
||||
state.Should().NotBeNull();
|
||||
state!.DocumentDigests.Should().Contain(d => d.StartsWith("quarantine:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_ReplayingSnapshot_SkipsDuplicateDocuments()
|
||||
{
|
||||
using var fixture = await ConnectorFixture.CreateAsync();
|
||||
|
||||
var firstSink = new InMemoryRawSink();
|
||||
var firstContext = fixture.CreateContext(firstSink);
|
||||
await CollectAsync(fixture.Connector.FetchAsync(firstContext, CancellationToken.None));
|
||||
|
||||
var secondSink = new InMemoryRawSink();
|
||||
var secondContext = fixture.CreateContext(secondSink);
|
||||
var secondRunDocuments = await CollectAsync(fixture.Connector.FetchAsync(secondContext, CancellationToken.None));
|
||||
|
||||
secondRunDocuments.Should().BeEmpty();
|
||||
secondSink.Documents.Should().BeEmpty();
|
||||
|
||||
var state = fixture.StateRepository.State;
|
||||
state.Should().NotBeNull();
|
||||
state!.DocumentDigests.Should().Contain(fixture.ExpectedDocumentDigest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_TrimsPersistedDigestHistory()
|
||||
{
|
||||
var existingDigests = Enumerable.Range(0, ConnectorFixture.MaxDigestHistory + 5)
|
||||
.Select(i => $"sha256:{i:X32}")
|
||||
.ToImmutableArray();
|
||||
var initialState = new VexConnectorState(
|
||||
"excititor:suse.rancher",
|
||||
DateTimeOffset.Parse("2025-10-18T00:00:00Z", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal),
|
||||
ImmutableArray.CreateBuilder<string>()
|
||||
.Add("checkpoint:cursor-old")
|
||||
.AddRange(existingDigests)
|
||||
.ToImmutable());
|
||||
|
||||
using var fixture = await ConnectorFixture.CreateAsync(initialState);
|
||||
|
||||
var sink = new InMemoryRawSink();
|
||||
var context = fixture.CreateContext(sink);
|
||||
await CollectAsync(fixture.Connector.FetchAsync(context, CancellationToken.None));
|
||||
|
||||
var state = fixture.StateRepository.State;
|
||||
state.Should().NotBeNull();
|
||||
state!.DocumentDigests.Should().Contain(d => d.StartsWith("checkpoint:", StringComparison.Ordinal));
|
||||
state.DocumentDigests.Count.Should().Be(ConnectorFixture.MaxDigestHistory + 1);
|
||||
}
|
||||
|
||||
private static async Task<List<VexRawDocument>> CollectAsync(IAsyncEnumerable<VexRawDocument> source)
|
||||
{
|
||||
var list = new List<VexRawDocument>();
|
||||
await foreach (var document in source.ConfigureAwait(false))
|
||||
{
|
||||
list.Add(document);
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
#region helpers
|
||||
|
||||
private sealed class ConnectorFixture : IDisposable
|
||||
{
|
||||
public const int MaxDigestHistory = 200;
|
||||
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly TempDirectory _tempDirectory;
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
private ConnectorFixture(
|
||||
RancherHubConnector connector,
|
||||
InMemoryConnectorStateRepository stateRepository,
|
||||
RoutingHttpMessageHandler handler,
|
||||
IServiceProvider serviceProvider,
|
||||
TempDirectory tempDirectory,
|
||||
HttpClient httpClient,
|
||||
Uri documentUri,
|
||||
string documentDigest)
|
||||
{
|
||||
Connector = connector;
|
||||
StateRepository = stateRepository;
|
||||
Handler = handler;
|
||||
_serviceProvider = serviceProvider;
|
||||
_tempDirectory = tempDirectory;
|
||||
_httpClient = httpClient;
|
||||
DocumentUri = documentUri;
|
||||
ExpectedDocumentDigest = $"sha256:{documentDigest}";
|
||||
}
|
||||
|
||||
public RancherHubConnector Connector { get; }
|
||||
|
||||
public InMemoryConnectorStateRepository StateRepository { get; }
|
||||
|
||||
public RoutingHttpMessageHandler Handler { get; }
|
||||
|
||||
public Uri DocumentUri { get; }
|
||||
|
||||
public string ExpectedDocumentDigest { get; }
|
||||
|
||||
public VexConnectorContext CreateContext(InMemoryRawSink sink, DateTimeOffset? since = null)
|
||||
=> new(
|
||||
since,
|
||||
VexConnectorSettings.Empty,
|
||||
sink,
|
||||
new NoopSignatureVerifier(),
|
||||
new NoopNormalizerRouter(),
|
||||
_serviceProvider,
|
||||
ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_httpClient.Dispose();
|
||||
_tempDirectory.Dispose();
|
||||
}
|
||||
|
||||
public static async Task<ConnectorFixture> CreateAsync(VexConnectorState? initialState = null)
|
||||
{
|
||||
var tempDirectory = new TempDirectory();
|
||||
var documentPayload = "{\"document\":\"payload\"}";
|
||||
var documentDigest = ComputeSha256Hex(documentPayload);
|
||||
|
||||
var documentUri = new Uri("https://hub.test/events/evt-1.json");
|
||||
var eventsPayload = """
|
||||
{
|
||||
"cursor": "cursor-1",
|
||||
"nextCursor": "cursor-2",
|
||||
"events": [
|
||||
{
|
||||
"id": "evt-1",
|
||||
"type": "vex.statement.published",
|
||||
"channel": "rancher/rke2",
|
||||
"publishedAt": "2025-10-19T12:00:00Z",
|
||||
"document": {
|
||||
"uri": "https://hub.test/events/evt-1.json",
|
||||
"sha256": "DOC_DIGEST",
|
||||
"format": "csaf"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
""".Replace("DOC_DIGEST", documentDigest, StringComparison.Ordinal);
|
||||
|
||||
var eventsPath = tempDirectory.Combine("events.json");
|
||||
await File.WriteAllTextAsync(eventsPath, eventsPayload, Encoding.UTF8).ConfigureAwait(false);
|
||||
var eventsChecksum = ComputeSha256Hex(eventsPayload);
|
||||
|
||||
var discoveryPayload = """
|
||||
{
|
||||
"hubId": "excititor:suse.rancher",
|
||||
"title": "SUSE Rancher VEX Hub",
|
||||
"subscription": {
|
||||
"eventsUri": "https://hub.test/events",
|
||||
"checkpointUri": "https://hub.test/checkpoint",
|
||||
"channels": [ "rancher/rke2" ],
|
||||
"requiresAuthentication": false
|
||||
},
|
||||
"offline": {
|
||||
"snapshotUri": "EVENTS_URI",
|
||||
"sha256": "EVENTS_DIGEST"
|
||||
}
|
||||
}
|
||||
"""
|
||||
.Replace("EVENTS_URI", new Uri(eventsPath).ToString(), StringComparison.Ordinal)
|
||||
.Replace("EVENTS_DIGEST", eventsChecksum, StringComparison.Ordinal);
|
||||
|
||||
var discoveryPath = tempDirectory.Combine("discovery.json");
|
||||
await File.WriteAllTextAsync(discoveryPath, discoveryPayload, Encoding.UTF8).ConfigureAwait(false);
|
||||
|
||||
var handler = new RoutingHttpMessageHandler();
|
||||
handler.SetRoute(documentUri, () => JsonResponse(documentPayload));
|
||||
var httpClient = new HttpClient(handler)
|
||||
{
|
||||
Timeout = TimeSpan.FromSeconds(10),
|
||||
};
|
||||
var httpFactory = new SingletonHttpClientFactory(httpClient);
|
||||
|
||||
var memoryCache = new MemoryCache(new MemoryCacheOptions());
|
||||
var fileSystem = new System.IO.Abstractions.FileSystem();
|
||||
var tokenProvider = new RancherHubTokenProvider(httpFactory, memoryCache, NullLogger<RancherHubTokenProvider>.Instance);
|
||||
var metadataLoader = new RancherHubMetadataLoader(httpFactory, memoryCache, tokenProvider, fileSystem, NullLogger<RancherHubMetadataLoader>.Instance);
|
||||
var eventClient = new RancherHubEventClient(httpFactory, tokenProvider, fileSystem, NullLogger<RancherHubEventClient>.Instance);
|
||||
|
||||
var stateRepository = new InMemoryConnectorStateRepository(initialState);
|
||||
var checkpointManager = new RancherHubCheckpointManager(stateRepository);
|
||||
|
||||
var validators = new[] { new RancherHubConnectorOptionsValidator(fileSystem) };
|
||||
var connector = new RancherHubConnector(
|
||||
metadataLoader,
|
||||
eventClient,
|
||||
checkpointManager,
|
||||
tokenProvider,
|
||||
httpFactory,
|
||||
NullLogger<RancherHubConnector>.Instance,
|
||||
TimeProvider.System,
|
||||
validators);
|
||||
|
||||
state.Should().NotBeNull();
|
||||
state!.DocumentDigests.Should().Contain(d => d.StartsWith("quarantine:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_ReplayingSnapshot_SkipsDuplicateDocuments()
|
||||
{
|
||||
using var fixture = await ConnectorFixture.CreateAsync();
|
||||
|
||||
var firstSink = new InMemoryRawSink();
|
||||
var firstContext = fixture.CreateContext(firstSink);
|
||||
await CollectAsync(fixture.Connector.FetchAsync(firstContext, CancellationToken.None));
|
||||
|
||||
var secondSink = new InMemoryRawSink();
|
||||
var secondContext = fixture.CreateContext(secondSink);
|
||||
var secondRunDocuments = await CollectAsync(fixture.Connector.FetchAsync(secondContext, CancellationToken.None));
|
||||
|
||||
secondRunDocuments.Should().BeEmpty();
|
||||
secondSink.Documents.Should().BeEmpty();
|
||||
|
||||
var state = fixture.StateRepository.State;
|
||||
state.Should().NotBeNull();
|
||||
state!.DocumentDigests.Should().Contain(fixture.ExpectedDocumentDigest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchAsync_TrimsPersistedDigestHistory()
|
||||
{
|
||||
var existingDigests = Enumerable.Range(0, ConnectorFixture.MaxDigestHistory + 5)
|
||||
.Select(i => $"sha256:{i:X32}")
|
||||
.ToImmutableArray();
|
||||
var initialState = new VexConnectorState(
|
||||
"excititor:suse.rancher",
|
||||
DateTimeOffset.Parse("2025-10-18T00:00:00Z", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal),
|
||||
ImmutableArray.CreateBuilder<string>()
|
||||
.Add("checkpoint:cursor-old")
|
||||
.AddRange(existingDigests)
|
||||
.ToImmutable());
|
||||
|
||||
using var fixture = await ConnectorFixture.CreateAsync(initialState);
|
||||
|
||||
var sink = new InMemoryRawSink();
|
||||
var context = fixture.CreateContext(sink);
|
||||
await CollectAsync(fixture.Connector.FetchAsync(context, CancellationToken.None));
|
||||
|
||||
var state = fixture.StateRepository.State;
|
||||
state.Should().NotBeNull();
|
||||
state!.DocumentDigests.Should().Contain(d => d.StartsWith("checkpoint:", StringComparison.Ordinal));
|
||||
state.DocumentDigests.Count.Should().Be(ConnectorFixture.MaxDigestHistory + 1);
|
||||
}
|
||||
|
||||
private static async Task<List<VexRawDocument>> CollectAsync(IAsyncEnumerable<VexRawDocument> source)
|
||||
{
|
||||
var list = new List<VexRawDocument>();
|
||||
await foreach (var document in source.ConfigureAwait(false))
|
||||
{
|
||||
list.Add(document);
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
#region helpers
|
||||
|
||||
private sealed class ConnectorFixture : IDisposable
|
||||
{
|
||||
public const int MaxDigestHistory = 200;
|
||||
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly TempDirectory _tempDirectory;
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
private ConnectorFixture(
|
||||
RancherHubConnector connector,
|
||||
InMemoryConnectorStateRepository stateRepository,
|
||||
RoutingHttpMessageHandler handler,
|
||||
IServiceProvider serviceProvider,
|
||||
TempDirectory tempDirectory,
|
||||
HttpClient httpClient,
|
||||
Uri documentUri,
|
||||
string documentDigest)
|
||||
{
|
||||
Connector = connector;
|
||||
StateRepository = stateRepository;
|
||||
Handler = handler;
|
||||
_serviceProvider = serviceProvider;
|
||||
_tempDirectory = tempDirectory;
|
||||
_httpClient = httpClient;
|
||||
DocumentUri = documentUri;
|
||||
ExpectedDocumentDigest = $"sha256:{documentDigest}";
|
||||
}
|
||||
|
||||
public RancherHubConnector Connector { get; }
|
||||
|
||||
public InMemoryConnectorStateRepository StateRepository { get; }
|
||||
|
||||
public RoutingHttpMessageHandler Handler { get; }
|
||||
|
||||
public Uri DocumentUri { get; }
|
||||
|
||||
public string ExpectedDocumentDigest { get; }
|
||||
|
||||
public VexConnectorContext CreateContext(InMemoryRawSink sink, DateTimeOffset? since = null)
|
||||
=> new(
|
||||
since,
|
||||
VexConnectorSettings.Empty,
|
||||
sink,
|
||||
new NoopSignatureVerifier(),
|
||||
new NoopNormalizerRouter(),
|
||||
_serviceProvider,
|
||||
ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_httpClient.Dispose();
|
||||
_tempDirectory.Dispose();
|
||||
}
|
||||
|
||||
public static async Task<ConnectorFixture> CreateAsync(VexConnectorState? initialState = null)
|
||||
{
|
||||
var tempDirectory = new TempDirectory();
|
||||
var documentPayload = "{\"document\":\"payload\"}";
|
||||
var documentDigest = ComputeSha256Hex(documentPayload);
|
||||
|
||||
var documentUri = new Uri("https://hub.test/events/evt-1.json");
|
||||
var eventsPayload = """
|
||||
{
|
||||
"cursor": "cursor-1",
|
||||
"nextCursor": "cursor-2",
|
||||
"events": [
|
||||
{
|
||||
"id": "evt-1",
|
||||
"type": "vex.statement.published",
|
||||
"channel": "rancher/rke2",
|
||||
"publishedAt": "2025-10-19T12:00:00Z",
|
||||
"document": {
|
||||
"uri": "https://hub.test/events/evt-1.json",
|
||||
"sha256": "DOC_DIGEST",
|
||||
"format": "csaf"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
""".Replace("DOC_DIGEST", documentDigest, StringComparison.Ordinal);
|
||||
|
||||
var eventsPath = tempDirectory.Combine("events.json");
|
||||
await File.WriteAllTextAsync(eventsPath, eventsPayload, Encoding.UTF8).ConfigureAwait(false);
|
||||
var eventsChecksum = ComputeSha256Hex(eventsPayload);
|
||||
|
||||
var discoveryPayload = """
|
||||
{
|
||||
"hubId": "excititor:suse.rancher",
|
||||
"title": "SUSE Rancher VEX Hub",
|
||||
"subscription": {
|
||||
"eventsUri": "https://hub.test/events",
|
||||
"checkpointUri": "https://hub.test/checkpoint",
|
||||
"channels": [ "rancher/rke2" ],
|
||||
"requiresAuthentication": false
|
||||
},
|
||||
"offline": {
|
||||
"snapshotUri": "EVENTS_URI",
|
||||
"sha256": "EVENTS_DIGEST"
|
||||
}
|
||||
}
|
||||
"""
|
||||
.Replace("EVENTS_URI", new Uri(eventsPath).ToString(), StringComparison.Ordinal)
|
||||
.Replace("EVENTS_DIGEST", eventsChecksum, StringComparison.Ordinal);
|
||||
|
||||
var discoveryPath = tempDirectory.Combine("discovery.json");
|
||||
await File.WriteAllTextAsync(discoveryPath, discoveryPayload, Encoding.UTF8).ConfigureAwait(false);
|
||||
|
||||
var handler = new RoutingHttpMessageHandler();
|
||||
handler.SetRoute(documentUri, () => JsonResponse(documentPayload));
|
||||
var httpClient = new HttpClient(handler)
|
||||
{
|
||||
Timeout = TimeSpan.FromSeconds(10),
|
||||
};
|
||||
var httpFactory = new SingletonHttpClientFactory(httpClient);
|
||||
|
||||
var memoryCache = new MemoryCache(new MemoryCacheOptions());
|
||||
var fileSystem = new System.IO.Abstractions.FileSystem();
|
||||
var tokenProvider = new RancherHubTokenProvider(httpFactory, memoryCache, NullLogger<RancherHubTokenProvider>.Instance);
|
||||
var metadataLoader = new RancherHubMetadataLoader(httpFactory, memoryCache, tokenProvider, fileSystem, NullLogger<RancherHubMetadataLoader>.Instance);
|
||||
var eventClient = new RancherHubEventClient(httpFactory, tokenProvider, fileSystem, NullLogger<RancherHubEventClient>.Instance);
|
||||
|
||||
var stateRepository = new InMemoryConnectorStateRepository(initialState);
|
||||
var checkpointManager = new RancherHubCheckpointManager(stateRepository);
|
||||
|
||||
var validators = new[] { new RancherHubConnectorOptionsValidator(fileSystem) };
|
||||
var connector = new RancherHubConnector(
|
||||
metadataLoader,
|
||||
eventClient,
|
||||
checkpointManager,
|
||||
tokenProvider,
|
||||
httpFactory,
|
||||
NullLogger<RancherHubConnector>.Instance,
|
||||
TimeProvider.System,
|
||||
validators);
|
||||
|
||||
var settingsValues = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
settingsValues["DiscoveryUri"] = "https://hub.test/.well-known/rancher-hub.json";
|
||||
settingsValues["OfflineSnapshotPath"] = discoveryPath;
|
||||
@@ -289,160 +288,160 @@ public sealed class RancherHubConnectorTests
|
||||
settingsValues["PgpFingerprints:0"] = "AABBCCDDEEFF00112233445566778899AABBCCDD";
|
||||
settingsValues["PgpFingerprints:1"] = "11223344556677889900AABBCCDDEEFF00112233";
|
||||
var settings = new VexConnectorSettings(settingsValues.ToImmutable());
|
||||
await connector.ValidateAsync(settings, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
var services = new ServiceCollection().BuildServiceProvider();
|
||||
|
||||
return new ConnectorFixture(
|
||||
connector,
|
||||
stateRepository,
|
||||
handler,
|
||||
services,
|
||||
tempDirectory,
|
||||
httpClient,
|
||||
documentUri,
|
||||
documentDigest);
|
||||
}
|
||||
|
||||
private static HttpResponseMessage JsonResponse(string payload)
|
||||
{
|
||||
var response = new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(payload, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SingletonHttpClientFactory : IHttpClientFactory
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public SingletonHttpClientFactory(HttpClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public HttpClient CreateClient(string name) => _client;
|
||||
}
|
||||
|
||||
private sealed class RoutingHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Dictionary<Uri, Queue<Func<HttpResponseMessage>>> _routes = new();
|
||||
|
||||
public void SetRoute(Uri uri, params Func<HttpResponseMessage>[] responders)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(uri);
|
||||
if (responders is null || responders.Length == 0)
|
||||
{
|
||||
_routes.Remove(uri);
|
||||
return;
|
||||
}
|
||||
|
||||
_routes[uri] = new Queue<Func<HttpResponseMessage>>(responders);
|
||||
}
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (request.RequestUri is not null &&
|
||||
_routes.TryGetValue(request.RequestUri, out var queue) &&
|
||||
queue.Count > 0)
|
||||
{
|
||||
var responder = queue.Count > 1 ? queue.Dequeue() : queue.Peek();
|
||||
var response = responder();
|
||||
response.RequestMessage = request;
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
|
||||
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
|
||||
{
|
||||
Content = new StringContent($"No response configured for {request.RequestUri}", Encoding.UTF8, "text/plain"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
public InMemoryConnectorStateRepository(VexConnectorState? initialState = null)
|
||||
{
|
||||
State = initialState;
|
||||
}
|
||||
|
||||
public VexConnectorState? State { get; private set; }
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult(State);
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
{
|
||||
State = state;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryRawSink : IVexRawDocumentSink
|
||||
{
|
||||
public List<VexRawDocument> Documents { get; } = new();
|
||||
|
||||
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
Documents.Add(document);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
|
||||
private sealed class TempDirectory : IDisposable
|
||||
{
|
||||
private readonly string _path;
|
||||
|
||||
public TempDirectory()
|
||||
{
|
||||
_path = Path.Combine(Path.GetTempPath(), "stellaops-excititor-tests", Guid.NewGuid().ToString("n"));
|
||||
Directory.CreateDirectory(_path);
|
||||
}
|
||||
|
||||
public string Combine(string relative) => Path.Combine(_path, relative);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_path))
|
||||
{
|
||||
Directory.Delete(_path, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256Hex(string payload)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(payload);
|
||||
return ComputeSha256Hex(bytes);
|
||||
}
|
||||
|
||||
private static string ComputeSha256Hex(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[32];
|
||||
SHA256.HashData(payload, buffer);
|
||||
return Convert.ToHexString(buffer).ToLowerInvariant();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
await connector.ValidateAsync(settings, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
var services = new ServiceCollection().BuildServiceProvider();
|
||||
|
||||
return new ConnectorFixture(
|
||||
connector,
|
||||
stateRepository,
|
||||
handler,
|
||||
services,
|
||||
tempDirectory,
|
||||
httpClient,
|
||||
documentUri,
|
||||
documentDigest);
|
||||
}
|
||||
|
||||
private static HttpResponseMessage JsonResponse(string payload)
|
||||
{
|
||||
var response = new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(payload, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SingletonHttpClientFactory : IHttpClientFactory
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public SingletonHttpClientFactory(HttpClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public HttpClient CreateClient(string name) => _client;
|
||||
}
|
||||
|
||||
private sealed class RoutingHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Dictionary<Uri, Queue<Func<HttpResponseMessage>>> _routes = new();
|
||||
|
||||
public void SetRoute(Uri uri, params Func<HttpResponseMessage>[] responders)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(uri);
|
||||
if (responders is null || responders.Length == 0)
|
||||
{
|
||||
_routes.Remove(uri);
|
||||
return;
|
||||
}
|
||||
|
||||
_routes[uri] = new Queue<Func<HttpResponseMessage>>(responders);
|
||||
}
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (request.RequestUri is not null &&
|
||||
_routes.TryGetValue(request.RequestUri, out var queue) &&
|
||||
queue.Count > 0)
|
||||
{
|
||||
var responder = queue.Count > 1 ? queue.Dequeue() : queue.Peek();
|
||||
var response = responder();
|
||||
response.RequestMessage = request;
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
|
||||
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)
|
||||
{
|
||||
Content = new StringContent($"No response configured for {request.RequestUri}", Encoding.UTF8, "text/plain"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
public InMemoryConnectorStateRepository(VexConnectorState? initialState = null)
|
||||
{
|
||||
State = initialState;
|
||||
}
|
||||
|
||||
public VexConnectorState? State { get; private set; }
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(State);
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
State = state;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryRawSink : IVexRawDocumentSink
|
||||
{
|
||||
public List<VexRawDocument> Documents { get; } = new();
|
||||
|
||||
public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
Documents.Add(document);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
|
||||
private sealed class TempDirectory : IDisposable
|
||||
{
|
||||
private readonly string _path;
|
||||
|
||||
public TempDirectory()
|
||||
{
|
||||
_path = Path.Combine(Path.GetTempPath(), "stellaops-excititor-tests", Guid.NewGuid().ToString("n"));
|
||||
Directory.CreateDirectory(_path);
|
||||
}
|
||||
|
||||
public string Combine(string relative) => Path.Combine(_path, relative);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_path))
|
||||
{
|
||||
Directory.Delete(_path, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256Hex(string payload)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(payload);
|
||||
return ComputeSha256Hex(bytes);
|
||||
}
|
||||
|
||||
private static string ComputeSha256Hex(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[32];
|
||||
SHA256.HashData(payload, buffer);
|
||||
return Convert.ToHexString(buffer).ToLowerInvariant();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Remove="..\..\..\StellaOps.Concelier.Tests.Shared\AssemblyInfo.cs" />
|
||||
|
||||
@@ -17,10 +17,8 @@ using StellaOps.Excititor.Connectors.Ubuntu.CSAF;
|
||||
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration;
|
||||
using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using System.IO.Abstractions.TestingHelpers;
|
||||
using Xunit;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors;
|
||||
|
||||
@@ -374,10 +372,10 @@ public sealed class UbuntuCsafConnectorTests
|
||||
{
|
||||
public VexConnectorState? CurrentState { get; private set; }
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(CurrentState);
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
CurrentState = state;
|
||||
return ValueTask.CompletedTask;
|
||||
@@ -399,13 +397,13 @@ public sealed class UbuntuCsafConnectorTests
|
||||
{
|
||||
public List<VexProvider> SavedProviders { get; } = new();
|
||||
|
||||
public ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(SavedProviders.LastOrDefault(provider => provider.Id == id));
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexProvider>>(SavedProviders.ToList());
|
||||
|
||||
public ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken)
|
||||
{
|
||||
var existingIndex = SavedProviders.FindIndex(p => p.Id == provider.Id);
|
||||
if (existingIndex >= 0)
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" PrivateAssets="all" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
<ProjectReference Include="../../StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
||||
@@ -5,9 +5,7 @@ using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using Xunit;
|
||||
|
||||
@@ -86,10 +84,10 @@ public sealed class VexEvidenceChunkServiceTests
|
||||
_claims = claims;
|
||||
}
|
||||
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken)
|
||||
{
|
||||
var query = _claims
|
||||
.Where(claim => claim.VulnerabilityId == vulnerabilityId)
|
||||
|
||||
@@ -4,12 +4,10 @@ using System.IO;
|
||||
using System.Text;
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Attestation.Verification;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.Export.Tests;
|
||||
@@ -212,14 +210,14 @@ public sealed class ExportEngineTests
|
||||
|
||||
public VexExportManifest? LastSavedManifest { get; private set; }
|
||||
|
||||
public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
{
|
||||
var key = CreateKey(signature.Value, format);
|
||||
_store.TryGetValue(key, out var manifest);
|
||||
return ValueTask.FromResult<VexExportManifest?>(manifest);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
var key = CreateKey(manifest.QuerySignature.Value, manifest.Format);
|
||||
_store[key] = manifest;
|
||||
@@ -299,13 +297,13 @@ public sealed class ExportEngineTests
|
||||
{
|
||||
public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new();
|
||||
|
||||
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexCacheEntry?>(null);
|
||||
|
||||
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
{
|
||||
RemoveCalls[(signature.Value, format)] = true;
|
||||
return ValueTask.CompletedTask;
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.Export.Tests;
|
||||
|
||||
@@ -53,13 +51,13 @@ public sealed class VexExportCacheServiceTests
|
||||
public VexExportFormat LastFormat { get; private set; }
|
||||
public int RemoveCalls { get; private set; }
|
||||
|
||||
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexCacheEntry?>(null);
|
||||
|
||||
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
{
|
||||
LastSignature = signature;
|
||||
LastFormat = format;
|
||||
@@ -73,10 +71,10 @@ public sealed class VexExportCacheServiceTests
|
||||
public int ExpiredCount { get; set; }
|
||||
public int DanglingCount { get; set; }
|
||||
|
||||
public ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(ExpiredCount);
|
||||
|
||||
public ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(DanglingCount);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public MongoVexCacheMaintenanceTests()
|
||||
{
|
||||
_database = _mongo.CreateDatabase("cache-maintenance");
|
||||
VexMongoMappingRegistry.Register();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemoveExpiredAsync_DeletesEntriesBeforeCutoff()
|
||||
{
|
||||
var collection = _database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache);
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
await collection.InsertManyAsync(new[]
|
||||
{
|
||||
new VexCacheEntryRecord
|
||||
{
|
||||
Id = "sig-1|json",
|
||||
QuerySignature = "sig-1",
|
||||
Format = "json",
|
||||
ArtifactAlgorithm = "sha256",
|
||||
ArtifactDigest = "deadbeef",
|
||||
CreatedAt = now.AddHours(-2),
|
||||
ExpiresAt = now.AddHours(-1),
|
||||
},
|
||||
new VexCacheEntryRecord
|
||||
{
|
||||
Id = "sig-2|json",
|
||||
QuerySignature = "sig-2",
|
||||
Format = "json",
|
||||
ArtifactAlgorithm = "sha256",
|
||||
ArtifactDigest = "cafebabe",
|
||||
CreatedAt = now,
|
||||
ExpiresAt = now.AddHours(1),
|
||||
},
|
||||
});
|
||||
|
||||
var maintenance = new MongoVexCacheMaintenance(_database, NullLogger<MongoVexCacheMaintenance>.Instance);
|
||||
var removed = await maintenance.RemoveExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, removed);
|
||||
|
||||
var remaining = await collection.CountDocumentsAsync(FilterDefinition<VexCacheEntryRecord>.Empty);
|
||||
Assert.Equal(1, remaining);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemoveMissingManifestReferencesAsync_DropsDanglingEntries()
|
||||
{
|
||||
var cache = _database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache);
|
||||
var exports = _database.GetCollection<VexExportManifestRecord>(VexMongoCollectionNames.Exports);
|
||||
|
||||
await exports.InsertOneAsync(new VexExportManifestRecord
|
||||
{
|
||||
Id = "manifest-existing",
|
||||
QuerySignature = "sig-keep",
|
||||
Format = "json",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ArtifactAlgorithm = "sha256",
|
||||
ArtifactDigest = "keep",
|
||||
ClaimCount = 1,
|
||||
SourceProviders = new List<string> { "vendor" },
|
||||
});
|
||||
|
||||
await cache.InsertManyAsync(new[]
|
||||
{
|
||||
new VexCacheEntryRecord
|
||||
{
|
||||
Id = "sig-remove|json",
|
||||
QuerySignature = "sig-remove",
|
||||
Format = "json",
|
||||
ArtifactAlgorithm = "sha256",
|
||||
ArtifactDigest = "drop",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ManifestId = "manifest-missing",
|
||||
},
|
||||
new VexCacheEntryRecord
|
||||
{
|
||||
Id = "sig-keep|json",
|
||||
QuerySignature = "sig-keep",
|
||||
Format = "json",
|
||||
ArtifactAlgorithm = "sha256",
|
||||
ArtifactDigest = "keep",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ManifestId = "manifest-existing",
|
||||
},
|
||||
});
|
||||
|
||||
var maintenance = new MongoVexCacheMaintenance(_database, NullLogger<MongoVexCacheMaintenance>.Instance);
|
||||
var removed = await maintenance.RemoveMissingManifestReferencesAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, removed);
|
||||
|
||||
var remainingIds = await cache.Find(Builders<VexCacheEntryRecord>.Filter.Empty)
|
||||
.Project(x => x.Id)
|
||||
.ToListAsync();
|
||||
Assert.Single(remainingIds);
|
||||
Assert.Contains("sig-keep|json", remainingIds);
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Aoc;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexRepositoryTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly MongoClient _client;
|
||||
|
||||
public MongoVexRepositoryTests()
|
||||
{
|
||||
_client = _mongo.Client;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RawStore_UsesGridFsForLargePayloads()
|
||||
{
|
||||
var database = _mongo.CreateDatabase("vex-raw-gridfs");
|
||||
var store = CreateRawStore(database, thresholdBytes: 32);
|
||||
|
||||
var payload = CreateJsonPayload(new string('A', 256));
|
||||
var document = new VexRawDocument(
|
||||
"red-hat",
|
||||
VexDocumentFormat.Csaf,
|
||||
new Uri("https://example.com/redhat/csaf.json"),
|
||||
DateTimeOffset.UtcNow,
|
||||
"sha256:large",
|
||||
payload,
|
||||
ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
await store.StoreAsync(document, CancellationToken.None);
|
||||
|
||||
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var stored = await rawCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", document.Digest))
|
||||
.FirstOrDefaultAsync();
|
||||
|
||||
Assert.NotNull(stored);
|
||||
Assert.True(stored!.TryGetValue("GridFsObjectId", out var gridId));
|
||||
Assert.False(gridId.IsBsonNull);
|
||||
Assert.Empty(stored["Content"].AsBsonBinaryData.Bytes);
|
||||
|
||||
var filesCollection = database.GetCollection<BsonDocument>("vex.raw.files");
|
||||
var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
|
||||
Assert.Equal(1, fileCount);
|
||||
|
||||
var fetched = await store.FindByDigestAsync(document.Digest, CancellationToken.None);
|
||||
Assert.NotNull(fetched);
|
||||
Assert.Equal(payload, fetched!.Content.ToArray());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RawStore_ReplacesGridFsWithInlinePayload()
|
||||
{
|
||||
var database = _mongo.CreateDatabase("vex-raw-inline");
|
||||
var store = CreateRawStore(database, thresholdBytes: 16);
|
||||
|
||||
var largePayload = CreateJsonPayload(new string('B', 128));
|
||||
var digest = "sha256:inline";
|
||||
var largeDocument = new VexRawDocument(
|
||||
"cisco",
|
||||
VexDocumentFormat.CycloneDx,
|
||||
new Uri("https://example.com/cyclonedx.json"),
|
||||
DateTimeOffset.UtcNow,
|
||||
digest,
|
||||
largePayload,
|
||||
ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
await store.StoreAsync(largeDocument, CancellationToken.None);
|
||||
|
||||
var smallDocument = largeDocument with
|
||||
{
|
||||
RetrievedAt = DateTimeOffset.UtcNow.AddMinutes(1),
|
||||
Content = CreateJsonPayload("small"),
|
||||
};
|
||||
|
||||
await store.StoreAsync(smallDocument, CancellationToken.None);
|
||||
|
||||
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var stored = await rawCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", digest))
|
||||
.FirstOrDefaultAsync();
|
||||
|
||||
Assert.NotNull(stored);
|
||||
Assert.True(stored!.TryGetValue("GridFsObjectId", out var gridId));
|
||||
Assert.True(gridId.IsBsonNull);
|
||||
var storedContent = Encoding.UTF8.GetString(stored["Content"].AsBsonBinaryData.Bytes);
|
||||
Assert.Equal(CreateJsonPayloadString("small"), storedContent);
|
||||
|
||||
var filesCollection = database.GetCollection<BsonDocument>("vex.raw.files");
|
||||
var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
|
||||
Assert.Equal(0, fileCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RawStore_WhenGuardRejectsDocument_DoesNotPersist()
|
||||
{
|
||||
var database = _client.GetDatabase($"vex-raw-guard-{Guid.NewGuid():N}");
|
||||
var guard = new RecordingVexRawWriteGuard { ShouldThrow = true };
|
||||
var store = CreateRawStore(database, thresholdBytes: 64, guard);
|
||||
|
||||
var payload = CreateJsonPayload("guard-check");
|
||||
var document = new VexRawDocument(
|
||||
"vendor.guard",
|
||||
VexDocumentFormat.Csaf,
|
||||
new Uri("https://example.com/guard.json"),
|
||||
DateTimeOffset.UtcNow,
|
||||
"sha256:guard",
|
||||
payload,
|
||||
ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
await Assert.ThrowsAsync<ExcititorAocGuardException>(() => store.StoreAsync(document, CancellationToken.None).AsTask());
|
||||
|
||||
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var count = await rawCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
|
||||
Assert.Equal(0, count);
|
||||
Assert.NotNull(guard.LastDocument);
|
||||
Assert.Equal("tenant-default", guard.LastDocument!.Tenant);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExportStore_SavesManifestAndCacheTransactionally()
|
||||
{
|
||||
var database = _client.GetDatabase($"vex-export-save-{Guid.NewGuid():N}");
|
||||
var options = Options.Create(new VexMongoStorageOptions
|
||||
{
|
||||
ExportCacheTtl = TimeSpan.FromHours(6),
|
||||
GridFsInlineThresholdBytes = 64,
|
||||
});
|
||||
|
||||
var sessionProvider = new VexMongoSessionProvider(_client, options);
|
||||
var store = new MongoVexExportStore(_client, database, options, sessionProvider);
|
||||
var signature = new VexQuerySignature("format=csaf|provider=redhat");
|
||||
var manifest = new VexExportManifest(
|
||||
"exports/20251016/redhat",
|
||||
signature,
|
||||
VexExportFormat.Csaf,
|
||||
DateTimeOffset.UtcNow,
|
||||
new VexContentAddress("sha256", "abcdef123456"),
|
||||
claimCount: 5,
|
||||
sourceProviders: new[] { "red-hat" },
|
||||
fromCache: false,
|
||||
consensusRevision: "rev-1",
|
||||
attestation: null,
|
||||
sizeBytes: 1024);
|
||||
|
||||
await store.SaveAsync(manifest, CancellationToken.None);
|
||||
|
||||
var exportsCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Exports);
|
||||
var exportKey = BuildExportKey(signature, VexExportFormat.Csaf);
|
||||
var exportDoc = await exportsCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", exportKey))
|
||||
.FirstOrDefaultAsync();
|
||||
Assert.NotNull(exportDoc);
|
||||
|
||||
var cacheCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Cache);
|
||||
var cacheKey = BuildExportKey(signature, VexExportFormat.Csaf);
|
||||
var cacheDoc = await cacheCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", cacheKey))
|
||||
.FirstOrDefaultAsync();
|
||||
|
||||
Assert.NotNull(cacheDoc);
|
||||
Assert.Equal(manifest.ExportId, cacheDoc!["ManifestId"].AsString);
|
||||
Assert.True(cacheDoc.TryGetValue("ExpiresAt", out var expiresValue));
|
||||
Assert.False(expiresValue.IsBsonNull);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExportStore_FindAsync_ExpiresCacheEntries()
|
||||
{
|
||||
var database = _mongo.CreateDatabase("vex-export-expire");
|
||||
var options = Options.Create(new VexMongoStorageOptions
|
||||
{
|
||||
ExportCacheTtl = TimeSpan.FromMinutes(5),
|
||||
GridFsInlineThresholdBytes = 64,
|
||||
});
|
||||
|
||||
var sessionProvider = new VexMongoSessionProvider(_client, options);
|
||||
var store = new MongoVexExportStore(_client, database, options, sessionProvider);
|
||||
var signature = new VexQuerySignature("format=json|provider=cisco");
|
||||
var manifest = new VexExportManifest(
|
||||
"exports/20251016/cisco",
|
||||
signature,
|
||||
VexExportFormat.Json,
|
||||
DateTimeOffset.UtcNow,
|
||||
new VexContentAddress("sha256", "deadbeef"),
|
||||
claimCount: 3,
|
||||
sourceProviders: new[] { "cisco" },
|
||||
fromCache: false,
|
||||
consensusRevision: "rev-2",
|
||||
attestation: null,
|
||||
sizeBytes: 2048);
|
||||
|
||||
await store.SaveAsync(manifest, CancellationToken.None);
|
||||
|
||||
var cacheCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Cache);
|
||||
var cacheId = BuildExportKey(signature, VexExportFormat.Json);
|
||||
var update = Builders<BsonDocument>.Update.Set("ExpiresAt", DateTime.UtcNow.AddMinutes(-10));
|
||||
await cacheCollection.UpdateOneAsync(Builders<BsonDocument>.Filter.Eq("_id", cacheId), update);
|
||||
|
||||
var cached = await store.FindAsync(signature, VexExportFormat.Json, CancellationToken.None);
|
||||
Assert.Null(cached);
|
||||
|
||||
var remaining = await cacheCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", cacheId))
|
||||
.FirstOrDefaultAsync();
|
||||
Assert.Null(remaining);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ClaimStore_AppendsAndQueriesStatements()
|
||||
{
|
||||
var database = _mongo.CreateDatabase("vex-claims");
|
||||
var store = new MongoVexClaimStore(database);
|
||||
|
||||
var product = new VexProduct("pkg:demo/app", "Demo App", version: "1.0.0", purl: "pkg:demo/app@1.0.0");
|
||||
var document = new VexClaimDocument(
|
||||
VexDocumentFormat.Csaf,
|
||||
"sha256:claim-1",
|
||||
new Uri("https://example.org/vex/claim-1.json"),
|
||||
revision: "2025-10-19");
|
||||
|
||||
var initialClaim = new VexClaim(
|
||||
vulnerabilityId: "CVE-2025-0101",
|
||||
providerId: "redhat",
|
||||
product: product,
|
||||
status: VexClaimStatus.NotAffected,
|
||||
document: document,
|
||||
firstSeen: DateTimeOffset.UtcNow.AddMinutes(-30),
|
||||
lastSeen: DateTimeOffset.UtcNow.AddMinutes(-10),
|
||||
justification: VexJustification.ComponentNotPresent,
|
||||
detail: "Package not shipped in this channel.",
|
||||
confidence: new VexConfidence("high", 0.9, "policy/default"),
|
||||
signals: new VexSignalSnapshot(
|
||||
new VexSeveritySignal("CVSS:3.1", 5.8, "medium", "CVSS:3.1/..."),
|
||||
kev: false,
|
||||
epss: 0.21),
|
||||
additionalMetadata: ImmutableDictionary<string, string>.Empty.Add("source", "csaf"));
|
||||
|
||||
await store.AppendAsync(new[] { initialClaim }, DateTimeOffset.UtcNow.AddMinutes(-5), CancellationToken.None);
|
||||
|
||||
var secondDocument = new VexClaimDocument(
|
||||
VexDocumentFormat.Csaf,
|
||||
"sha256:claim-2",
|
||||
new Uri("https://example.org/vex/claim-2.json"),
|
||||
revision: "2025-10-19.1");
|
||||
|
||||
var secondClaim = new VexClaim(
|
||||
vulnerabilityId: initialClaim.VulnerabilityId,
|
||||
providerId: initialClaim.ProviderId,
|
||||
product: initialClaim.Product,
|
||||
status: initialClaim.Status,
|
||||
document: secondDocument,
|
||||
firstSeen: initialClaim.FirstSeen,
|
||||
lastSeen: DateTimeOffset.UtcNow,
|
||||
justification: initialClaim.Justification,
|
||||
detail: initialClaim.Detail,
|
||||
confidence: initialClaim.Confidence,
|
||||
signals: new VexSignalSnapshot(
|
||||
new VexSeveritySignal("CVSS:3.1", 7.2, "high"),
|
||||
kev: true,
|
||||
epss: 0.43),
|
||||
additionalMetadata: initialClaim.AdditionalMetadata.ToImmutableDictionary(kvp => kvp.Key, kvp => kvp.Value));
|
||||
|
||||
await store.AppendAsync(new[] { secondClaim }, DateTimeOffset.UtcNow, CancellationToken.None);
|
||||
|
||||
var all = await store.FindAsync("CVE-2025-0101", product.Key, since: null, CancellationToken.None);
|
||||
var allList = all.ToList();
|
||||
Assert.Equal(2, allList.Count);
|
||||
Assert.Equal("sha256:claim-2", allList[0].Document.Digest);
|
||||
Assert.True(allList[0].Signals?.Kev);
|
||||
Assert.Equal(0.43, allList[0].Signals?.Epss);
|
||||
Assert.Equal("sha256:claim-1", allList[1].Document.Digest);
|
||||
Assert.Equal("csaf", allList[1].AdditionalMetadata["source"]);
|
||||
|
||||
var recentOnly = await store.FindAsync("CVE-2025-0101", product.Key, DateTimeOffset.UtcNow.AddMinutes(-2), CancellationToken.None);
|
||||
var recentList = recentOnly.ToList();
|
||||
Assert.Single(recentList);
|
||||
Assert.Equal("sha256:claim-2", recentList[0].Document.Digest);
|
||||
}
|
||||
|
||||
private MongoVexRawStore CreateRawStore(IMongoDatabase database, int thresholdBytes, IVexRawWriteGuard? guard = null)
|
||||
{
|
||||
var options = Options.Create(new VexMongoStorageOptions
|
||||
{
|
||||
RawBucketName = "vex.raw",
|
||||
GridFsInlineThresholdBytes = thresholdBytes,
|
||||
ExportCacheTtl = TimeSpan.FromHours(1),
|
||||
});
|
||||
|
||||
var sessionProvider = new VexMongoSessionProvider(_client, options);
|
||||
var guardInstance = guard ?? new PassthroughVexRawWriteGuard();
|
||||
return new MongoVexRawStore(_client, database, options, sessionProvider, guardInstance);
|
||||
}
|
||||
|
||||
private static string BuildExportKey(VexQuerySignature signature, VexExportFormat format)
|
||||
=> string.Format(CultureInfo.InvariantCulture, "{0}|{1}", signature.Value, format.ToString().ToLowerInvariant());
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
|
||||
private static byte[] CreateJsonPayload(string value)
|
||||
=> Encoding.UTF8.GetBytes(CreateJsonPayloadString(value));
|
||||
|
||||
private static string CreateJsonPayloadString(string value)
|
||||
=> $"{{\"data\":\"{value}\"}}";
|
||||
|
||||
private sealed class RecordingVexRawWriteGuard : IVexRawWriteGuard
|
||||
{
|
||||
public bool ShouldThrow { get; set; }
|
||||
|
||||
public RawVexDocumentModel? LastDocument { get; private set; }
|
||||
|
||||
public void EnsureValid(RawVexDocumentModel document)
|
||||
{
|
||||
LastDocument = document;
|
||||
if (ShouldThrow)
|
||||
{
|
||||
var violation = AocViolation.Create(AocViolationCode.InvalidTenant, "/tenant", "Guard rejected document.");
|
||||
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PassthroughVexRawWriteGuard : IVexRawWriteGuard
|
||||
{
|
||||
public void EnsureValid(RawVexDocumentModel document)
|
||||
{
|
||||
// No-op guard for unit tests.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly MongoClient _client;
|
||||
|
||||
public MongoVexSessionConsistencyTests()
|
||||
{
|
||||
_client = _mongo.Client;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionProvidesReadYourWrites()
|
||||
{
|
||||
await using var provider = BuildServiceProvider();
|
||||
await using var scope = provider.CreateAsyncScope();
|
||||
|
||||
var sessionProvider = scope.ServiceProvider.GetRequiredService<IVexMongoSessionProvider>();
|
||||
var providerStore = scope.ServiceProvider.GetRequiredService<IVexProviderStore>();
|
||||
|
||||
var session = await sessionProvider.StartSessionAsync();
|
||||
var descriptor = new VexProvider("red-hat", "Red Hat", VexProviderKind.Vendor);
|
||||
|
||||
await providerStore.SaveAsync(descriptor, CancellationToken.None, session);
|
||||
var fetched = await providerStore.FindAsync(descriptor.Id, CancellationToken.None, session);
|
||||
|
||||
Assert.NotNull(fetched);
|
||||
Assert.Equal(descriptor.DisplayName, fetched!.DisplayName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionMaintainsMonotonicReadsAcrossStepDown()
|
||||
{
|
||||
await using var provider = BuildServiceProvider();
|
||||
await using var scope = provider.CreateAsyncScope();
|
||||
|
||||
var client = scope.ServiceProvider.GetRequiredService<IMongoClient>();
|
||||
var sessionProvider = scope.ServiceProvider.GetRequiredService<IVexMongoSessionProvider>();
|
||||
var providerStore = scope.ServiceProvider.GetRequiredService<IVexProviderStore>();
|
||||
|
||||
var session = await sessionProvider.StartSessionAsync();
|
||||
var initial = new VexProvider("cisco", "Cisco", VexProviderKind.Vendor);
|
||||
|
||||
await providerStore.SaveAsync(initial, CancellationToken.None, session);
|
||||
var baseline = await providerStore.FindAsync(initial.Id, CancellationToken.None, session);
|
||||
Assert.Equal("Cisco", baseline!.DisplayName);
|
||||
|
||||
await ForcePrimaryStepDownAsync(client, CancellationToken.None);
|
||||
await WaitForPrimaryAsync(client, CancellationToken.None);
|
||||
|
||||
await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
var updated = new VexProvider(initial.Id, "Cisco Systems", initial.Kind);
|
||||
await providerStore.SaveAsync(updated, CancellationToken.None, session);
|
||||
}, CancellationToken.None);
|
||||
|
||||
var afterFailover = await providerStore.FindAsync(initial.Id, CancellationToken.None, session);
|
||||
Assert.Equal("Cisco Systems", afterFailover!.DisplayName);
|
||||
|
||||
var subsequent = await providerStore.FindAsync(initial.Id, CancellationToken.None, session);
|
||||
Assert.Equal("Cisco Systems", subsequent!.DisplayName);
|
||||
}
|
||||
|
||||
private ServiceProvider BuildServiceProvider()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddDebug());
|
||||
services.Configure<VexMongoStorageOptions>(options =>
|
||||
{
|
||||
options.ConnectionString = _mongo.ConnectionString;
|
||||
options.DatabaseName = _mongo.ReserveDatabase("session");
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
options.RawBucketName = "vex.raw";
|
||||
});
|
||||
services.AddExcititorMongoStorage();
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static async Task ExecuteWithRetryAsync(Func<Task> action, CancellationToken cancellationToken)
|
||||
{
|
||||
const int maxAttempts = 10;
|
||||
var attempt = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
await action();
|
||||
return;
|
||||
}
|
||||
catch (MongoException ex) when (IsStepDownTransient(ex) && attempt++ < maxAttempts)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsStepDownTransient(MongoException ex)
|
||||
{
|
||||
if (ex is MongoConnectionException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (ex is MongoCommandException command)
|
||||
{
|
||||
return command.Code is 7 or 89 or 91 or 10107 or 11600
|
||||
|| string.Equals(command.CodeName, "NotPrimaryNoSecondaryOk", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(command.CodeName, "NotWritablePrimary", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(command.CodeName, "PrimarySteppedDown", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(command.CodeName, "NotPrimary", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static async Task ForcePrimaryStepDownAsync(IMongoClient client, CancellationToken cancellationToken)
|
||||
{
|
||||
var admin = client.GetDatabase("admin");
|
||||
var command = new BsonDocument
|
||||
{
|
||||
{ "replSetStepDown", 1 },
|
||||
{ "force", true },
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
await admin.RunCommandAsync<BsonDocument>(command, cancellationToken: cancellationToken);
|
||||
}
|
||||
catch (MongoException ex) when (IsStepDownTransient(ex))
|
||||
{
|
||||
// Expected when the primary closes connections during the step-down sequence.
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WaitForPrimaryAsync(IMongoClient client, CancellationToken cancellationToken)
|
||||
{
|
||||
var admin = client.GetDatabase("admin");
|
||||
var helloCommand = new BsonDocument("hello", 1);
|
||||
|
||||
for (var attempt = 0; attempt < 40; attempt++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var result = await admin.RunCommandAsync<BsonDocument>(helloCommand, cancellationToken: cancellationToken);
|
||||
if (result.TryGetValue("isWritablePrimary", out var value) && value.IsBoolean && value.AsBoolean)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch (MongoException ex) when (IsStepDownTransient(ex))
|
||||
{
|
||||
// Primary still recovering, retry.
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken);
|
||||
}
|
||||
|
||||
throw new TimeoutException("Replica set primary did not recover in time.");
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
}
|
||||
@@ -1,182 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System.Text;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
|
||||
public MongoVexStatementBackfillServiceTests()
|
||||
{
|
||||
// Intentionally left blank; Mongo environment is initialized on demand.
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_BackfillsStatementsFromRawDocuments()
|
||||
{
|
||||
await using var provider = BuildServiceProvider();
|
||||
await using var scope = provider.CreateAsyncScope();
|
||||
|
||||
var rawStore = scope.ServiceProvider.GetRequiredService<IVexRawStore>();
|
||||
var claimStore = scope.ServiceProvider.GetRequiredService<IVexClaimStore>();
|
||||
var backfill = scope.ServiceProvider.GetRequiredService<VexStatementBackfillService>();
|
||||
|
||||
var retrievedAt = DateTimeOffset.UtcNow.AddMinutes(-15);
|
||||
var metadata = ImmutableDictionary<string, string>.Empty
|
||||
.Add("vulnId", "CVE-2025-0001")
|
||||
.Add("productKey", "pkg:test/app");
|
||||
|
||||
var document = new VexRawDocument(
|
||||
"test-provider",
|
||||
VexDocumentFormat.Csaf,
|
||||
new Uri("https://example.test/vex.json"),
|
||||
retrievedAt,
|
||||
"sha256:test-doc",
|
||||
CreateJsonPayload("backfill-1"),
|
||||
metadata);
|
||||
|
||||
await rawStore.StoreAsync(document, CancellationToken.None);
|
||||
|
||||
var result = await backfill.RunAsync(new VexStatementBackfillRequest(), CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, result.DocumentsEvaluated);
|
||||
Assert.Equal(1, result.DocumentsBackfilled);
|
||||
Assert.Equal(1, result.ClaimsWritten);
|
||||
Assert.Equal(0, result.NormalizationFailures);
|
||||
|
||||
var claims = await claimStore.FindAsync("CVE-2025-0001", "pkg:test/app", since: null, CancellationToken.None);
|
||||
var claim = Assert.Single(claims);
|
||||
Assert.Equal(VexClaimStatus.NotAffected, claim.Status);
|
||||
Assert.Equal("test-provider", claim.ProviderId);
|
||||
Assert.Equal(retrievedAt.ToUnixTimeSeconds(), claim.FirstSeen.ToUnixTimeSeconds());
|
||||
Assert.NotNull(claim.Signals);
|
||||
Assert.Equal(0.2, claim.Signals!.Epss);
|
||||
Assert.Equal("cvss", claim.Signals!.Severity?.Scheme);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_SkipsExistingDocumentsUnlessForced()
|
||||
{
|
||||
await using var provider = BuildServiceProvider();
|
||||
await using var scope = provider.CreateAsyncScope();
|
||||
|
||||
var rawStore = scope.ServiceProvider.GetRequiredService<IVexRawStore>();
|
||||
var claimStore = scope.ServiceProvider.GetRequiredService<IVexClaimStore>();
|
||||
var backfill = scope.ServiceProvider.GetRequiredService<VexStatementBackfillService>();
|
||||
|
||||
var metadata = ImmutableDictionary<string, string>.Empty
|
||||
.Add("vulnId", "CVE-2025-0002")
|
||||
.Add("productKey", "pkg:test/api");
|
||||
|
||||
var document = new VexRawDocument(
|
||||
"test-provider",
|
||||
VexDocumentFormat.Csaf,
|
||||
new Uri("https://example.test/vex-2.json"),
|
||||
DateTimeOffset.UtcNow.AddMinutes(-10),
|
||||
"sha256:test-doc-2",
|
||||
CreateJsonPayload("backfill-2"),
|
||||
metadata);
|
||||
|
||||
await rawStore.StoreAsync(document, CancellationToken.None);
|
||||
|
||||
var first = await backfill.RunAsync(new VexStatementBackfillRequest(), CancellationToken.None);
|
||||
Assert.Equal(1, first.DocumentsBackfilled);
|
||||
|
||||
var second = await backfill.RunAsync(new VexStatementBackfillRequest(), CancellationToken.None);
|
||||
Assert.Equal(1, second.DocumentsEvaluated);
|
||||
Assert.Equal(0, second.DocumentsBackfilled);
|
||||
Assert.Equal(1, second.SkippedExisting);
|
||||
|
||||
var forced = await backfill.RunAsync(new VexStatementBackfillRequest(Force: true), CancellationToken.None);
|
||||
Assert.Equal(1, forced.DocumentsBackfilled);
|
||||
|
||||
var claims = await claimStore.FindAsync("CVE-2025-0002", "pkg:test/api", since: null, CancellationToken.None);
|
||||
Assert.Equal(2, claims.Count);
|
||||
}
|
||||
|
||||
private ServiceProvider BuildServiceProvider()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddDebug());
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.Configure<VexMongoStorageOptions>(options =>
|
||||
{
|
||||
options.ConnectionString = _mongo.ConnectionString;
|
||||
options.DatabaseName = _mongo.ReserveDatabase("backfill");
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
options.RawBucketName = "vex.raw";
|
||||
options.GridFsInlineThresholdBytes = 1024;
|
||||
options.ExportCacheTtl = TimeSpan.FromHours(1);
|
||||
options.DefaultTenant = "tests";
|
||||
});
|
||||
services.AddExcititorMongoStorage();
|
||||
services.AddExcititorAocGuards();
|
||||
services.AddSingleton<IVexRawWriteGuard, PermissiveVexRawWriteGuard>();
|
||||
services.AddSingleton<IVexNormalizer, TestNormalizer>();
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
|
||||
private static ReadOnlyMemory<byte> CreateJsonPayload(string value)
|
||||
=> Encoding.UTF8.GetBytes($"{{\"data\":\"{value}\"}}");
|
||||
|
||||
private sealed class TestNormalizer : IVexNormalizer
|
||||
{
|
||||
public string Format => "csaf";
|
||||
|
||||
public bool CanHandle(VexRawDocument document) => true;
|
||||
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken)
|
||||
{
|
||||
var productKey = document.Metadata.TryGetValue("productKey", out var value) ? value : "pkg:test/default";
|
||||
var vulnId = document.Metadata.TryGetValue("vulnId", out var vuln) ? vuln : "CVE-TEST-0000";
|
||||
|
||||
var product = new VexProduct(productKey, "Test Product");
|
||||
var claimDocument = new VexClaimDocument(
|
||||
document.Format,
|
||||
document.Digest,
|
||||
document.SourceUri);
|
||||
|
||||
var timestamp = document.RetrievedAt == default ? DateTimeOffset.UtcNow : document.RetrievedAt;
|
||||
|
||||
var claim = new VexClaim(
|
||||
vulnId,
|
||||
provider.Id,
|
||||
product,
|
||||
VexClaimStatus.NotAffected,
|
||||
claimDocument,
|
||||
timestamp,
|
||||
timestamp,
|
||||
VexJustification.ComponentNotPresent,
|
||||
detail: "backfill-test",
|
||||
confidence: new VexConfidence("high", 0.95, "unit-test"),
|
||||
signals: new VexSignalSnapshot(
|
||||
new VexSeveritySignal("cvss", 5.4, "medium"),
|
||||
kev: false,
|
||||
epss: 0.2));
|
||||
|
||||
var claims = ImmutableArray.Create(claim);
|
||||
return ValueTask.FromResult(new VexClaimBatch(document, claims, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PermissiveVexRawWriteGuard : IVexRawWriteGuard
|
||||
{
|
||||
public void EnsureValid(RawVexDocumentModel document)
|
||||
{
|
||||
// Tests control the payloads; guard bypass keeps focus on backfill logic.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,260 +0,0 @@
|
||||
using System.Globalization;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class MongoVexStoreMappingTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public MongoVexStoreMappingTests()
|
||||
{
|
||||
_database = _mongo.CreateDatabase("storage-mapping");
|
||||
VexMongoMappingRegistry.Register();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProviderStore_RoundTrips_WithExtraFields()
|
||||
{
|
||||
var providers = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Providers);
|
||||
var providerId = "red-hat";
|
||||
|
||||
var document = new BsonDocument
|
||||
{
|
||||
{ "_id", providerId },
|
||||
{ "DisplayName", "Red Hat CSAF" },
|
||||
{ "Kind", "vendor" },
|
||||
{ "BaseUris", new BsonArray { "https://example.com/csaf" } },
|
||||
{
|
||||
"Discovery",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "WellKnownMetadata", "https://example.com/.well-known/csaf" },
|
||||
{ "RolIeService", "https://example.com/service/rolie" },
|
||||
{ "UnsupportedField", "ignored" },
|
||||
}
|
||||
},
|
||||
{
|
||||
"Trust",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Weight", 0.75 },
|
||||
{
|
||||
"Cosign",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Issuer", "issuer@example.com" },
|
||||
{ "IdentityPattern", "spiffe://example/*" },
|
||||
{ "Unexpected", true },
|
||||
}
|
||||
},
|
||||
{ "PgpFingerprints", new BsonArray { "ABCDEF1234567890" } },
|
||||
{ "AnotherIgnoredField", 123 },
|
||||
}
|
||||
},
|
||||
{ "Enabled", true },
|
||||
{ "UnexpectedRoot", new BsonDocument { { "flag", true } } },
|
||||
};
|
||||
|
||||
await providers.InsertOneAsync(document);
|
||||
|
||||
var store = new MongoVexProviderStore(_database);
|
||||
var result = await store.FindAsync(providerId, CancellationToken.None);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(providerId, result!.Id);
|
||||
Assert.Equal("Red Hat CSAF", result.DisplayName);
|
||||
Assert.Equal(VexProviderKind.Vendor, result.Kind);
|
||||
Assert.Single(result.BaseUris);
|
||||
Assert.Equal("https://example.com/csaf", result.BaseUris[0].ToString());
|
||||
Assert.Equal("https://example.com/.well-known/csaf", result.Discovery.WellKnownMetadata?.ToString());
|
||||
Assert.Equal("https://example.com/service/rolie", result.Discovery.RolIeService?.ToString());
|
||||
Assert.Equal(0.75, result.Trust.Weight);
|
||||
Assert.NotNull(result.Trust.Cosign);
|
||||
Assert.Equal("issuer@example.com", result.Trust.Cosign!.Issuer);
|
||||
Assert.Equal("spiffe://example/*", result.Trust.Cosign!.IdentityPattern);
|
||||
Assert.Contains("ABCDEF1234567890", result.Trust.PgpFingerprints);
|
||||
Assert.True(result.Enabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsensusStore_IgnoresUnknownFields()
|
||||
{
|
||||
var consensus = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
|
||||
var vulnerabilityId = "CVE-2025-12345";
|
||||
var productKey = "pkg:maven/org.example/app@1.2.3";
|
||||
var consensusId = string.Format(CultureInfo.InvariantCulture, "{0}|{1}", vulnerabilityId.Trim(), productKey.Trim());
|
||||
|
||||
var document = new BsonDocument
|
||||
{
|
||||
{ "_id", consensusId },
|
||||
{ "VulnerabilityId", vulnerabilityId },
|
||||
{
|
||||
"Product",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Key", productKey },
|
||||
{ "Name", "Example App" },
|
||||
{ "Version", "1.2.3" },
|
||||
{ "Purl", productKey },
|
||||
{ "Extra", "ignored" },
|
||||
}
|
||||
},
|
||||
{ "Status", "notaffected" },
|
||||
{ "CalculatedAt", DateTime.UtcNow },
|
||||
{
|
||||
"Sources",
|
||||
new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "ProviderId", "red-hat" },
|
||||
{ "Status", "notaffected" },
|
||||
{ "DocumentDigest", "sha256:123" },
|
||||
{ "Weight", 0.9 },
|
||||
{ "Justification", "componentnotpresent" },
|
||||
{ "Detail", "Vendor statement" },
|
||||
{
|
||||
"Confidence",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Level", "high" },
|
||||
{ "Score", 0.7 },
|
||||
{ "Method", "review" },
|
||||
{ "Unexpected", "ignored" },
|
||||
}
|
||||
},
|
||||
{ "UnknownField", true },
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"Conflicts",
|
||||
new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "ProviderId", "cisco" },
|
||||
{ "Status", "affected" },
|
||||
{ "DocumentDigest", "sha256:999" },
|
||||
{ "Justification", "requiresconfiguration" },
|
||||
{ "Detail", "Different guidance" },
|
||||
{ "Reason", "policy_override" },
|
||||
{ "Other", 1 },
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"Signals",
|
||||
new BsonDocument
|
||||
{
|
||||
{
|
||||
"Severity",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Scheme", "CVSS:3.1" },
|
||||
{ "Score", 7.5 },
|
||||
{ "Label", "high" },
|
||||
{ "Vector", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" },
|
||||
}
|
||||
},
|
||||
{ "Kev", true },
|
||||
{ "Epss", 0.42 },
|
||||
}
|
||||
},
|
||||
{ "PolicyVersion", "2025.10" },
|
||||
{ "PolicyRevisionId", "rev-1" },
|
||||
{ "PolicyDigest", "sha256:abc" },
|
||||
{ "Summary", "Vendor confirms not affected." },
|
||||
{ "GeneratedAt", DateTime.UtcNow },
|
||||
{ "Unexpected", new BsonDocument { { "foo", "bar" } } },
|
||||
};
|
||||
|
||||
await consensus.InsertOneAsync(document);
|
||||
|
||||
var store = new MongoVexConsensusStore(_database);
|
||||
var result = await store.FindAsync(vulnerabilityId, productKey, CancellationToken.None);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(vulnerabilityId, result!.VulnerabilityId);
|
||||
Assert.Equal(productKey, result.Product.Key);
|
||||
Assert.Equal("Example App", result.Product.Name);
|
||||
Assert.Equal(VexConsensusStatus.NotAffected, result.Status);
|
||||
Assert.Single(result.Sources);
|
||||
var source = result.Sources[0];
|
||||
Assert.Equal("red-hat", source.ProviderId);
|
||||
Assert.Equal(VexClaimStatus.NotAffected, source.Status);
|
||||
Assert.Equal("sha256:123", source.DocumentDigest);
|
||||
Assert.Equal(0.9, source.Weight);
|
||||
Assert.Equal(VexJustification.ComponentNotPresent, source.Justification);
|
||||
Assert.NotNull(source.Confidence);
|
||||
Assert.Equal("high", source.Confidence!.Level);
|
||||
Assert.Equal(0.7, source.Confidence!.Score);
|
||||
Assert.Equal("review", source.Confidence!.Method);
|
||||
Assert.Single(result.Conflicts);
|
||||
var conflict = result.Conflicts[0];
|
||||
Assert.Equal("cisco", conflict.ProviderId);
|
||||
Assert.Equal(VexClaimStatus.Affected, conflict.Status);
|
||||
Assert.Equal(VexJustification.RequiresConfiguration, conflict.Justification);
|
||||
Assert.Equal("policy_override", conflict.Reason);
|
||||
Assert.Equal("Vendor confirms not affected.", result.Summary);
|
||||
Assert.Equal("2025.10", result.PolicyVersion);
|
||||
Assert.NotNull(result.Signals);
|
||||
Assert.True(result.Signals!.Kev);
|
||||
Assert.Equal(0.42, result.Signals.Epss);
|
||||
Assert.NotNull(result.Signals.Severity);
|
||||
Assert.Equal("CVSS:3.1", result.Signals.Severity!.Scheme);
|
||||
Assert.Equal(7.5, result.Signals.Severity.Score);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CacheIndex_RoundTripsGridFsMetadata()
|
||||
{
|
||||
var gridObjectId = ObjectId.GenerateNewId().ToString();
|
||||
var index = new MongoVexCacheIndex(_database);
|
||||
var signature = new VexQuerySignature("format=csaf|vendor=redhat");
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var expires = now.AddHours(12);
|
||||
var entry = new VexCacheEntry(
|
||||
signature,
|
||||
VexExportFormat.Csaf,
|
||||
new VexContentAddress("sha256", "abcdef123456"),
|
||||
now,
|
||||
sizeBytes: 1024,
|
||||
manifestId: "manifest-001",
|
||||
gridFsObjectId: gridObjectId,
|
||||
expiresAt: expires);
|
||||
|
||||
await index.SaveAsync(entry, CancellationToken.None);
|
||||
|
||||
var cacheId = string.Format(
|
||||
CultureInfo.InvariantCulture,
|
||||
"{0}|{1}",
|
||||
signature.Value,
|
||||
entry.Format.ToString().ToLowerInvariant());
|
||||
|
||||
var cache = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Cache);
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("_id", cacheId);
|
||||
var update = Builders<BsonDocument>.Update.Set("UnexpectedField", true);
|
||||
await cache.UpdateOneAsync(filter, update);
|
||||
|
||||
var roundTrip = await index.FindAsync(signature, VexExportFormat.Csaf, CancellationToken.None);
|
||||
|
||||
Assert.NotNull(roundTrip);
|
||||
Assert.Equal(entry.QuerySignature.Value, roundTrip!.QuerySignature.Value);
|
||||
Assert.Equal(entry.Format, roundTrip.Format);
|
||||
Assert.Equal(entry.Artifact.Digest, roundTrip.Artifact.Digest);
|
||||
Assert.Equal(entry.ManifestId, roundTrip.ManifestId);
|
||||
Assert.Equal(entry.GridFsObjectId, roundTrip.GridFsObjectId);
|
||||
Assert.Equal(entry.SizeBytes, roundTrip.SizeBytes);
|
||||
Assert.NotNull(roundTrip.ExpiresAt);
|
||||
Assert.Equal(expires.ToUnixTimeMilliseconds(), roundTrip.ExpiresAt!.Value.ToUnixTimeMilliseconds());
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />
|
||||
<ProjectReference Include="../../../Concelier/__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,88 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
internal sealed class TestMongoEnvironment : IAsyncLifetime
|
||||
{
|
||||
private const string Prefix = "exstor";
|
||||
private readonly MongoDbRunner? _runner;
|
||||
private readonly HashSet<string> _reservedDatabases = new(StringComparer.Ordinal);
|
||||
|
||||
public TestMongoEnvironment()
|
||||
{
|
||||
var overrideConnection = Environment.GetEnvironmentVariable("EXCITITOR_TEST_MONGO_URI");
|
||||
if (!string.IsNullOrWhiteSpace(overrideConnection))
|
||||
{
|
||||
ConnectionString = overrideConnection.Trim();
|
||||
Client = new MongoClient(ConnectionString);
|
||||
return;
|
||||
}
|
||||
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
ConnectionString = _runner.ConnectionString;
|
||||
Client = new MongoClient(ConnectionString);
|
||||
}
|
||||
|
||||
public MongoClient Client { get; }
|
||||
|
||||
public string ConnectionString { get; }
|
||||
|
||||
public string ReserveDatabase(string hint)
|
||||
{
|
||||
var baseName = string.IsNullOrWhiteSpace(hint) ? "db" : hint.ToLowerInvariant();
|
||||
var builder = new StringBuilder(baseName.Length);
|
||||
foreach (var ch in baseName)
|
||||
{
|
||||
builder.Append(char.IsLetterOrDigit(ch) ? ch : '_');
|
||||
}
|
||||
|
||||
var slug = builder.Length == 0 ? "db" : builder.ToString();
|
||||
var suffix = ObjectId.GenerateNewId().ToString();
|
||||
var maxSlugLength = Math.Max(1, 60 - Prefix.Length - suffix.Length - 2);
|
||||
if (slug.Length > maxSlugLength)
|
||||
{
|
||||
slug = slug[..maxSlugLength];
|
||||
}
|
||||
|
||||
var name = $"{Prefix}_{slug}_{suffix}";
|
||||
_reservedDatabases.Add(name);
|
||||
return name;
|
||||
}
|
||||
|
||||
public IMongoDatabase CreateDatabase(string hint)
|
||||
{
|
||||
var name = ReserveDatabase(hint);
|
||||
return Client.GetDatabase(name);
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
if (_runner is not null)
|
||||
{
|
||||
_runner.Dispose();
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var db in _reservedDatabases)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Client.DropDatabaseAsync(db);
|
||||
}
|
||||
catch (MongoException)
|
||||
{
|
||||
// best-effort cleanup when sharing a developer-managed instance.
|
||||
}
|
||||
}
|
||||
|
||||
_reservedDatabases.Clear();
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Tests;
|
||||
|
||||
public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime
|
||||
{
|
||||
private readonly TestMongoEnvironment _mongo = new();
|
||||
private readonly IMongoDatabase _database;
|
||||
|
||||
public VexMongoMigrationRunnerTests()
|
||||
{
|
||||
_database = _mongo.CreateDatabase("migrations");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_AppliesInitialIndexesOnce()
|
||||
{
|
||||
var migrations = new IVexMongoMigration[]
|
||||
{
|
||||
new VexInitialIndexMigration(),
|
||||
new VexConsensusSignalsMigration(),
|
||||
new VexObservationCollectionsMigration(),
|
||||
};
|
||||
var runner = new VexMongoMigrationRunner(_database, migrations, NullLogger<VexMongoMigrationRunner>.Instance);
|
||||
|
||||
await runner.RunAsync(CancellationToken.None);
|
||||
await runner.RunAsync(CancellationToken.None);
|
||||
|
||||
var appliedCollection = _database.GetCollection<VexMigrationRecord>(VexMongoCollectionNames.Migrations);
|
||||
var applied = await appliedCollection.Find(FilterDefinition<VexMigrationRecord>.Empty).ToListAsync();
|
||||
Assert.Equal(3, applied.Count);
|
||||
Assert.Equal(migrations.Select(m => m.Id).OrderBy(id => id, StringComparer.Ordinal), applied.Select(record => record.Id).OrderBy(id => id, StringComparer.Ordinal));
|
||||
|
||||
Assert.True(HasIndex(_database.GetCollection<VexRawDocumentRecord>(VexMongoCollectionNames.Raw), "ProviderId_1_Format_1_RetrievedAt_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexProviderRecord>(VexMongoCollectionNames.Providers), "Kind_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexConsensusRecord>(VexMongoCollectionNames.Consensus), "VulnerabilityId_1_Product.Key_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexConsensusRecord>(VexMongoCollectionNames.Consensus), "PolicyRevisionId_1_PolicyDigest_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexConsensusRecord>(VexMongoCollectionNames.Consensus), "PolicyRevisionId_1_CalculatedAt_-1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexExportManifestRecord>(VexMongoCollectionNames.Exports), "QuerySignature_1_Format_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache), "QuerySignature_1_Format_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache), "ExpiresAt_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexStatementRecord>(VexMongoCollectionNames.Statements), "VulnerabilityId_1_Product.Key_1_InsertedAt_-1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexStatementRecord>(VexMongoCollectionNames.Statements), "ProviderId_1_InsertedAt_-1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexStatementRecord>(VexMongoCollectionNames.Statements), "Document.Digest_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations), "Tenant_1_ObservationId_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations), "Tenant_1_VulnerabilityId_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations), "Tenant_1_ProductKey_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations), "Tenant_1_Document.Digest_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations), "Tenant_1_ProviderId_1_Status_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexLinksetRecord>(VexMongoCollectionNames.Linksets), "Tenant_1_LinksetId_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexLinksetRecord>(VexMongoCollectionNames.Linksets), "Tenant_1_VulnerabilityId_1"));
|
||||
Assert.True(HasIndex(_database.GetCollection<VexLinksetRecord>(VexMongoCollectionNames.Linksets), "Tenant_1_ProductKey_1"));
|
||||
}
|
||||
|
||||
private static bool HasIndex<TDocument>(IMongoCollection<TDocument> collection, string name)
|
||||
{
|
||||
var indexes = collection.Indexes.List().ToList();
|
||||
return indexes.Any(index => index["name"].AsString == name);
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => _mongo.DisposeAsync();
|
||||
}
|
||||
@@ -5,7 +5,6 @@ using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using Xunit;
|
||||
|
||||
@@ -7,7 +7,6 @@ using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using EphemeralMongo;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -22,21 +21,16 @@ public sealed class BatchIngestValidationTests : IDisposable
|
||||
{
|
||||
private const string Tenant = "tests";
|
||||
|
||||
private readonly IMongoRunner _runner;
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public BatchIngestValidationTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "vex_batch_tests",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = Tenant,
|
||||
["Excititor:Storage:DefaultTenant"] = Tenant,
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
@@ -121,7 +115,6 @@ public sealed class BatchIngestValidationTests : IDisposable
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
|
||||
private sealed class IngestionMetricListener : IDisposable
|
||||
|
||||
@@ -8,7 +8,6 @@ using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using StellaOps.Excititor.WebService.Options;
|
||||
using Xunit;
|
||||
|
||||
@@ -1,212 +1,203 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using EphemeralMongo;
|
||||
using MongoRunner = EphemeralMongo.MongoRunner;
|
||||
using MongoRunnerOptions = EphemeralMongo.MongoRunnerOptions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class MirrorEndpointsTests : IDisposable
|
||||
{
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
private readonly IMongoRunner _runner;
|
||||
|
||||
public MirrorEndpointsTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
var data = new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "mirror-tests",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Id"] = "primary",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:DisplayName"] = "Primary Mirror",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxIndexRequestsPerHour"] = "1000",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxDownloadRequestsPerHour"] = "1000",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Key"] = "consensus",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Format"] = "json",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:vulnId"] = "CVE-2025-0001",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:productKey"] = "pkg:test/demo",
|
||||
};
|
||||
|
||||
configuration.AddInMemoryCollection(data!);
|
||||
},
|
||||
configureServices: services =>
|
||||
{
|
||||
TestServiceOverrides.Apply(services);
|
||||
services.RemoveAll<IVexExportStore>();
|
||||
services.AddSingleton<IVexExportStore>(provider =>
|
||||
{
|
||||
var timeProvider = provider.GetRequiredService<TimeProvider>();
|
||||
return new FakeExportStore(timeProvider);
|
||||
});
|
||||
services.RemoveAll<IVexArtifactStore>();
|
||||
services.AddSingleton<IVexArtifactStore>(_ => new FakeArtifactStore());
|
||||
services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF"));
|
||||
services.AddSingleton<StellaOps.Excititor.Attestation.Signing.IVexSigner, FakeSigner>();
|
||||
services.AddSingleton<StellaOps.Excititor.Policy.IVexPolicyEvaluator, FakePolicyEvaluator>();
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ListDomains_ReturnsConfiguredDomain()
|
||||
{
|
||||
var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/excititor/mirror/domains");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
var domains = document.RootElement.GetProperty("domains");
|
||||
Assert.Equal(1, domains.GetArrayLength());
|
||||
Assert.Equal("primary", domains[0].GetProperty("id").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DomainIndex_ReturnsManifestMetadata()
|
||||
{
|
||||
var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/excititor/mirror/domains/primary/index");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
var exports = document.RootElement.GetProperty("exports");
|
||||
Assert.Equal(1, exports.GetArrayLength());
|
||||
var entry = exports[0];
|
||||
Assert.Equal("consensus", entry.GetProperty("exportKey").GetString());
|
||||
Assert.Equal("exports/20251019T000000000Z/abcdef", entry.GetProperty("exportId").GetString());
|
||||
var artifact = entry.GetProperty("artifact");
|
||||
Assert.Equal("sha256", artifact.GetProperty("algorithm").GetString());
|
||||
Assert.Equal("deadbeef", artifact.GetProperty("digest").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Download_ReturnsArtifactContent()
|
||||
{
|
||||
var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/excititor/mirror/domains/primary/exports/consensus/download");
|
||||
response.EnsureSuccessStatusCode();
|
||||
Assert.Equal("application/json", response.Content.Headers.ContentType?.MediaType);
|
||||
var payload = await response.Content.ReadAsStringAsync();
|
||||
Assert.Equal("{\"status\":\"ok\"}", payload);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
|
||||
private sealed class FakeExportStore : IVexExportStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexExportManifest> _manifests = new();
|
||||
|
||||
public FakeExportStore(TimeProvider timeProvider)
|
||||
{
|
||||
var filters = new[]
|
||||
{
|
||||
new VexQueryFilter("vulnId", "CVE-2025-0001"),
|
||||
new VexQueryFilter("productKey", "pkg:test/demo"),
|
||||
};
|
||||
|
||||
var query = VexQuery.Create(filters, Enumerable.Empty<VexQuerySort>());
|
||||
var signature = VexQuerySignature.FromQuery(query);
|
||||
var createdAt = new DateTimeOffset(2025, 10, 19, 0, 0, 0, TimeSpan.Zero);
|
||||
|
||||
var manifest = new VexExportManifest(
|
||||
"exports/20251019T000000000Z/abcdef",
|
||||
signature,
|
||||
VexExportFormat.Json,
|
||||
createdAt,
|
||||
new VexContentAddress("sha256", "deadbeef"),
|
||||
1,
|
||||
new[] { "primary" },
|
||||
fromCache: false,
|
||||
consensusRevision: "rev-1",
|
||||
attestation: new VexAttestationMetadata("https://stella-ops.org/attestations/vex-export"),
|
||||
sizeBytes: 16);
|
||||
|
||||
_manifests.TryAdd((signature.Value, VexExportFormat.Json), manifest);
|
||||
|
||||
// Seed artifact content for download test.
|
||||
FakeArtifactStore.Seed(manifest.Artifact, "{\"status\":\"ok\"}");
|
||||
}
|
||||
|
||||
public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_manifests.TryGetValue((signature.Value, format), out var manifest);
|
||||
return ValueTask.FromResult(manifest);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class FakeArtifactStore : IVexArtifactStore
|
||||
{
|
||||
private static readonly ConcurrentDictionary<VexContentAddress, byte[]> Content = new();
|
||||
|
||||
public static void Seed(VexContentAddress contentAddress, string payload)
|
||||
{
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(payload);
|
||||
Content[contentAddress] = bytes;
|
||||
}
|
||||
|
||||
public ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken)
|
||||
{
|
||||
Content[artifact.ContentAddress] = artifact.Content.ToArray();
|
||||
return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory://artifact", artifact.Content.Length, artifact.Metadata));
|
||||
}
|
||||
|
||||
public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
|
||||
{
|
||||
Content.TryRemove(contentAddress, out _);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Content.TryGetValue(contentAddress, out var bytes))
|
||||
{
|
||||
return ValueTask.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<Stream?>(new MemoryStream(bytes, writable: false));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeSigner : StellaOps.Excititor.Attestation.Signing.IVexSigner
|
||||
{
|
||||
public ValueTask<StellaOps.Excititor.Attestation.Signing.VexSignedPayload> SignAsync(ReadOnlyMemory<byte> payload, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new StellaOps.Excititor.Attestation.Signing.VexSignedPayload("signature", "key"));
|
||||
}
|
||||
|
||||
private sealed class FakePolicyEvaluator : StellaOps.Excititor.Policy.IVexPolicyEvaluator
|
||||
{
|
||||
public string Version => "test";
|
||||
|
||||
public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default;
|
||||
|
||||
public double GetProviderWeight(VexProvider provider) => 1.0;
|
||||
|
||||
public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason)
|
||||
{
|
||||
rejectionReason = null;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Policy;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class MirrorEndpointsTests : IDisposable
|
||||
{
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public MirrorEndpointsTests()
|
||||
{
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
var data = new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Id"] = "primary",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:DisplayName"] = "Primary Mirror",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxIndexRequestsPerHour"] = "1000",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxDownloadRequestsPerHour"] = "1000",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Key"] = "consensus",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Format"] = "json",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:vulnId"] = "CVE-2025-0001",
|
||||
[$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:productKey"] = "pkg:test/demo",
|
||||
};
|
||||
|
||||
configuration.AddInMemoryCollection(data!);
|
||||
},
|
||||
configureServices: services =>
|
||||
{
|
||||
TestServiceOverrides.Apply(services);
|
||||
services.RemoveAll<IVexExportStore>();
|
||||
services.AddSingleton<IVexExportStore>(provider =>
|
||||
{
|
||||
var timeProvider = provider.GetRequiredService<TimeProvider>();
|
||||
return new FakeExportStore(timeProvider);
|
||||
});
|
||||
services.RemoveAll<IVexArtifactStore>();
|
||||
services.AddSingleton<IVexArtifactStore>(_ => new FakeArtifactStore());
|
||||
services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF"));
|
||||
services.AddSingleton<StellaOps.Excititor.Attestation.Signing.IVexSigner, FakeSigner>();
|
||||
services.AddSingleton<StellaOps.Excititor.Policy.IVexPolicyEvaluator, FakePolicyEvaluator>();
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ListDomains_ReturnsConfiguredDomain()
|
||||
{
|
||||
var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/excititor/mirror/domains");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
var domains = document.RootElement.GetProperty("domains");
|
||||
Assert.Equal(1, domains.GetArrayLength());
|
||||
Assert.Equal("primary", domains[0].GetProperty("id").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DomainIndex_ReturnsManifestMetadata()
|
||||
{
|
||||
var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/excititor/mirror/domains/primary/index");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
var exports = document.RootElement.GetProperty("exports");
|
||||
Assert.Equal(1, exports.GetArrayLength());
|
||||
var entry = exports[0];
|
||||
Assert.Equal("consensus", entry.GetProperty("exportKey").GetString());
|
||||
Assert.Equal("exports/20251019T000000000Z/abcdef", entry.GetProperty("exportId").GetString());
|
||||
var artifact = entry.GetProperty("artifact");
|
||||
Assert.Equal("sha256", artifact.GetProperty("algorithm").GetString());
|
||||
Assert.Equal("deadbeef", artifact.GetProperty("digest").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Download_ReturnsArtifactContent()
|
||||
{
|
||||
var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/excititor/mirror/domains/primary/exports/consensus/download");
|
||||
response.EnsureSuccessStatusCode();
|
||||
Assert.Equal("application/json", response.Content.Headers.ContentType?.MediaType);
|
||||
var payload = await response.Content.ReadAsStringAsync();
|
||||
Assert.Equal("{\"status\":\"ok\"}", payload);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
}
|
||||
|
||||
private sealed class FakeExportStore : IVexExportStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexExportManifest> _manifests = new();
|
||||
|
||||
public FakeExportStore(TimeProvider timeProvider)
|
||||
{
|
||||
var filters = new[]
|
||||
{
|
||||
new VexQueryFilter("vulnId", "CVE-2025-0001"),
|
||||
new VexQueryFilter("productKey", "pkg:test/demo"),
|
||||
};
|
||||
|
||||
var query = VexQuery.Create(filters, Enumerable.Empty<VexQuerySort>());
|
||||
var signature = VexQuerySignature.FromQuery(query);
|
||||
var createdAt = new DateTimeOffset(2025, 10, 19, 0, 0, 0, TimeSpan.Zero);
|
||||
|
||||
var manifest = new VexExportManifest(
|
||||
"exports/20251019T000000000Z/abcdef",
|
||||
signature,
|
||||
VexExportFormat.Json,
|
||||
createdAt,
|
||||
new VexContentAddress("sha256", "deadbeef"),
|
||||
1,
|
||||
new[] { "primary" },
|
||||
fromCache: false,
|
||||
consensusRevision: "rev-1",
|
||||
attestation: new VexAttestationMetadata("https://stella-ops.org/attestations/vex-export"),
|
||||
sizeBytes: 16);
|
||||
|
||||
_manifests.TryAdd((signature.Value, VexExportFormat.Json), manifest);
|
||||
|
||||
// Seed artifact content for download test.
|
||||
FakeArtifactStore.Seed(manifest.Artifact, "{\"status\":\"ok\"}");
|
||||
}
|
||||
|
||||
public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
{
|
||||
_manifests.TryGetValue((signature.Value, format), out var manifest);
|
||||
return ValueTask.FromResult(manifest);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class FakeArtifactStore : IVexArtifactStore
|
||||
{
|
||||
private static readonly ConcurrentDictionary<VexContentAddress, byte[]> Content = new();
|
||||
|
||||
public static void Seed(VexContentAddress contentAddress, string payload)
|
||||
{
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(payload);
|
||||
Content[contentAddress] = bytes;
|
||||
}
|
||||
|
||||
public ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken)
|
||||
{
|
||||
Content[artifact.ContentAddress] = artifact.Content.ToArray();
|
||||
return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory://artifact", artifact.Content.Length, artifact.Metadata));
|
||||
}
|
||||
|
||||
public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
|
||||
{
|
||||
Content.TryRemove(contentAddress, out _);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Content.TryGetValue(contentAddress, out var bytes))
|
||||
{
|
||||
return ValueTask.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<Stream?>(new MemoryStream(bytes, writable: false));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeSigner : StellaOps.Excititor.Attestation.Signing.IVexSigner
|
||||
{
|
||||
public ValueTask<StellaOps.Excititor.Attestation.Signing.VexSignedPayload> SignAsync(ReadOnlyMemory<byte> payload, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new StellaOps.Excititor.Attestation.Signing.VexSignedPayload("signature", "key"));
|
||||
}
|
||||
|
||||
private sealed class FakePolicyEvaluator : StellaOps.Excititor.Policy.IVexPolicyEvaluator
|
||||
{
|
||||
public string Version => "test";
|
||||
|
||||
public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default;
|
||||
|
||||
public double GetProviderWeight(VexProvider provider) => 1.0;
|
||||
|
||||
public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason)
|
||||
{
|
||||
rejectionReason = null;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -11,12 +11,10 @@ using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using EphemeralMongo;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using Xunit;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
@@ -24,20 +22,15 @@ namespace StellaOps.Excititor.WebService.Tests;
|
||||
public sealed class ObservabilityEndpointTests : IDisposable
|
||||
{
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
private readonly IMongoRunner _runner;
|
||||
|
||||
public ObservabilityEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "excititor_obs_tests",
|
||||
["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw",
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
["Excititor:Observability:IngestWarningThreshold"] = "00:10:00",
|
||||
["Excititor:Observability:IngestCriticalThreshold"] = "00:30:00",
|
||||
["Excititor:Observability:SignatureWindow"] = "00:30:00",
|
||||
@@ -51,7 +44,7 @@ public sealed class ObservabilityEndpointTests : IDisposable
|
||||
services.AddTestAuthentication();
|
||||
|
||||
services.RemoveAll<IVexConnectorStateRepository>();
|
||||
services.AddScoped<IVexConnectorStateRepository, MongoVexConnectorStateRepository>();
|
||||
services.AddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
|
||||
services.AddSingleton<IVexConnector>(_ => new StubConnector("excititor:redhat", VexProviderKind.Distro));
|
||||
});
|
||||
|
||||
@@ -94,46 +87,15 @@ public sealed class ObservabilityEndpointTests : IDisposable
|
||||
private void SeedDatabase()
|
||||
{
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var database = scope.ServiceProvider.GetRequiredService<IMongoDatabase>();
|
||||
database.DropCollection(VexMongoCollectionNames.Raw);
|
||||
database.DropCollection(VexMongoCollectionNames.Consensus);
|
||||
database.DropCollection(VexMongoCollectionNames.ConnectorState);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
rawCollection.InsertMany(new[]
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Id", "raw-1" },
|
||||
{ "ProviderId", "excititor:redhat" },
|
||||
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
|
||||
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument { { "signature.present", "true" }, { "signature.verified", "true" } } }
|
||||
},
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Id", "raw-2" },
|
||||
{ "ProviderId", "excititor:redhat" },
|
||||
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
|
||||
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument { { "signature.present", "true" } } }
|
||||
},
|
||||
new BsonDocument
|
||||
{
|
||||
{ "Id", "raw-3" },
|
||||
{ "ProviderId", "excititor:redhat" },
|
||||
{ ObservabilityEndpointTestsHelper.RetrievedAtField, now },
|
||||
{ ObservabilityEndpointTestsHelper.MetadataField, new BsonDocument() }
|
||||
}
|
||||
});
|
||||
|
||||
var consensus = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
|
||||
consensus.InsertMany(new[]
|
||||
{
|
||||
ObservabilityEndpointTestsHelper.CreateConsensusDocument("c1", now, "affected"),
|
||||
ObservabilityEndpointTestsHelper.CreateConsensusDocument("c2", now.AddMinutes(-5), "not_affected")
|
||||
});
|
||||
|
||||
var rawStore = scope.ServiceProvider.GetRequiredService<IVexRawStore>();
|
||||
var linksetStore = scope.ServiceProvider.GetRequiredService<IAppendOnlyLinksetStore>();
|
||||
var providerStore = scope.ServiceProvider.GetRequiredService<IVexProviderStore>();
|
||||
var stateRepository = scope.ServiceProvider.GetRequiredService<IVexConnectorStateRepository>();
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var provider = new VexProvider("excititor:redhat", "Red Hat", VexProviderKind.Distro);
|
||||
providerStore.SaveAsync(provider, CancellationToken.None).GetAwaiter().GetResult();
|
||||
|
||||
var state = new VexConnectorState(
|
||||
"excititor:redhat",
|
||||
now.AddMinutes(-5),
|
||||
@@ -144,12 +106,67 @@ public sealed class ObservabilityEndpointTests : IDisposable
|
||||
now.AddMinutes(10),
|
||||
null);
|
||||
stateRepository.SaveAsync(state, CancellationToken.None).GetAwaiter().GetResult();
|
||||
|
||||
var metadataVerified = ImmutableDictionary<string, string>.Empty
|
||||
.Add("signature.present", "true")
|
||||
.Add("signature.verified", "true")
|
||||
.Add("tenant", "tests");
|
||||
var metadataUnsigned = ImmutableDictionary<string, string>.Empty
|
||||
.Add("signature.present", "true")
|
||||
.Add("tenant", "tests");
|
||||
var metadataMissing = ImmutableDictionary<string, string>.Empty.Add("tenant", "tests");
|
||||
|
||||
rawStore.StoreAsync(new VexRawDocument(
|
||||
"excititor:redhat",
|
||||
VexDocumentFormat.Csaf,
|
||||
new Uri("https://example.test/raw1.json"),
|
||||
now,
|
||||
"sha256:raw-1",
|
||||
"{\"stub\":\"payload\"}"u8.ToArray(),
|
||||
metadataVerified), CancellationToken.None).GetAwaiter().GetResult();
|
||||
|
||||
rawStore.StoreAsync(new VexRawDocument(
|
||||
"excititor:redhat",
|
||||
VexDocumentFormat.Csaf,
|
||||
new Uri("https://example.test/raw2.json"),
|
||||
now,
|
||||
"sha256:raw-2",
|
||||
"{\"stub\":\"payload\"}"u8.ToArray(),
|
||||
metadataUnsigned), CancellationToken.None).GetAwaiter().GetResult();
|
||||
|
||||
rawStore.StoreAsync(new VexRawDocument(
|
||||
"excititor:redhat",
|
||||
VexDocumentFormat.Csaf,
|
||||
new Uri("https://example.test/raw3.json"),
|
||||
now,
|
||||
"sha256:raw-3",
|
||||
"{\"stub\":\"payload\"}"u8.ToArray(),
|
||||
metadataMissing), CancellationToken.None).GetAwaiter().GetResult();
|
||||
|
||||
var scopeMetadata = new VexProductScope("pkg:test/demo", "demo", null, "pkg:test/demo", null, Array.Empty<string>());
|
||||
linksetStore.AppendObservationsBatchAsync(
|
||||
"tests",
|
||||
"CVE-2025-0001",
|
||||
"pkg:test/demo",
|
||||
new[]
|
||||
{
|
||||
new VexLinksetObservationRefModel("obs-1", "excititor:redhat", "affected", 0.9),
|
||||
new VexLinksetObservationRefModel("obs-2", "excititor:redhat", "fixed", 0.5)
|
||||
},
|
||||
scopeMetadata,
|
||||
CancellationToken.None).GetAwaiter().GetResult();
|
||||
|
||||
linksetStore.AppendDisagreementAsync(
|
||||
"tests",
|
||||
"CVE-2025-0001",
|
||||
"pkg:test/demo",
|
||||
new VexObservationDisagreement("excititor:redhat", "affected", "coverage-gap", 0.7),
|
||||
CancellationToken.None).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
|
||||
private sealed class StubConnector : IVexConnector
|
||||
@@ -177,32 +194,3 @@ public sealed class ObservabilityEndpointTests : IDisposable
|
||||
ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
internal static class ObservabilityEndpointTestsHelper
|
||||
{
|
||||
public const string RetrievedAtField = "RetrievedAt";
|
||||
public const string MetadataField = "Metadata";
|
||||
|
||||
public static BsonDocument CreateConsensusDocument(string id, DateTime timestamp, string conflictStatus)
|
||||
{
|
||||
var conflicts = new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "ProviderId", "excititor:redhat" },
|
||||
{ "Status", conflictStatus },
|
||||
{ "DocumentDigest", Guid.NewGuid().ToString("n") }
|
||||
}
|
||||
};
|
||||
|
||||
return new BsonDocument
|
||||
{
|
||||
{ "Id", id },
|
||||
{ "VulnerabilityId", $"CVE-{id}" },
|
||||
{ "Product", new BsonDocument { { "Key", $"pkg:{id}" }, { "Name", $"pkg-{id}" } } },
|
||||
{ "Status", "affected" },
|
||||
{ "CalculatedAt", timestamp },
|
||||
{ "Conflicts", conflicts }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,9 +6,6 @@ using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using EphemeralMongo;
|
||||
using MongoRunner = EphemeralMongo.MongoRunner;
|
||||
using MongoRunnerOptions = EphemeralMongo.MongoRunnerOptions;
|
||||
using StellaOps.Excititor.Attestation.Signing;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Policy;
|
||||
@@ -23,11 +20,9 @@ namespace StellaOps.Excititor.WebService.Tests;
|
||||
public sealed class OpenApiDiscoveryEndpointTests : IDisposable
|
||||
{
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
private readonly IMongoRunner _runner;
|
||||
|
||||
public OpenApiDiscoveryEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: config =>
|
||||
{
|
||||
@@ -35,10 +30,7 @@ public sealed class OpenApiDiscoveryEndpointTests : IDisposable
|
||||
Directory.CreateDirectory(rootPath);
|
||||
var settings = new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "excititor-openapi-tests",
|
||||
["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw",
|
||||
["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256",
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
["Excititor:Artifacts:FileSystem:RootPath"] = rootPath,
|
||||
};
|
||||
config.AddInMemoryCollection(settings!);
|
||||
@@ -173,7 +165,6 @@ public sealed class OpenApiDiscoveryEndpointTests : IDisposable
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
|
||||
private sealed class FakeSigner : IVexSigner
|
||||
|
||||
@@ -2,7 +2,6 @@ using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
@@ -86,13 +85,13 @@ public sealed class PolicyEndpointsTests
|
||||
_claims = claims;
|
||||
}
|
||||
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(_claims.Where(c => c.VulnerabilityId == vulnerabilityId && c.Product.Key == productKey).ToList());
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(_claims.Where(c => c.VulnerabilityId == vulnerabilityId).Take(limit).ToList());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,27 +5,20 @@ using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using EphemeralMongo;
|
||||
using MongoRunner = EphemeralMongo.MongoRunner;
|
||||
using MongoRunnerOptions = EphemeralMongo.MongoRunnerOptions;
|
||||
using StellaOps.Excititor.Attestation.Signing;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Policy;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class ResolveEndpointTests : IDisposable
|
||||
{
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
private readonly IMongoRunner _runner;
|
||||
|
||||
public ResolveEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: config =>
|
||||
{
|
||||
@@ -33,10 +26,7 @@ public sealed class ResolveEndpointTests : IDisposable
|
||||
Directory.CreateDirectory(rootPath);
|
||||
var settings = new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "excititor-resolve-tests",
|
||||
["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw",
|
||||
["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256",
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
["Excititor:Artifacts:FileSystem:RootPath"] = rootPath,
|
||||
};
|
||||
config.AddInMemoryCollection(settings!);
|
||||
@@ -197,7 +187,6 @@ public sealed class ResolveEndpointTests : IDisposable
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
|
||||
private sealed class ResolveRequest
|
||||
|
||||
@@ -5,9 +5,6 @@ using System.Net.Http.Json;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using EphemeralMongo;
|
||||
using MongoRunner = EphemeralMongo.MongoRunner;
|
||||
using MongoRunnerOptions = EphemeralMongo.MongoRunnerOptions;
|
||||
using StellaOps.Excititor.Attestation.Signing;
|
||||
using StellaOps.Excititor.Connectors.Abstractions;
|
||||
using StellaOps.Excititor.Policy;
|
||||
@@ -20,11 +17,9 @@ namespace StellaOps.Excititor.WebService.Tests;
|
||||
public sealed class StatusEndpointTests : IDisposable
|
||||
{
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
private readonly IMongoRunner _runner;
|
||||
|
||||
public StatusEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: config =>
|
||||
{
|
||||
@@ -32,10 +27,9 @@ public sealed class StatusEndpointTests : IDisposable
|
||||
Directory.CreateDirectory(rootPath);
|
||||
var settings = new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "excititor-web-tests",
|
||||
["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw",
|
||||
["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256",
|
||||
["Postgres:Excititor:ConnectionString"] = "Host=localhost;Username=postgres;Password=postgres;Database=excititor_tests",
|
||||
["Postgres:Excititor:SchemaName"] = "vex",
|
||||
["Excititor:Storage:InlineThresholdBytes"] = "256",
|
||||
["Excititor:Artifacts:FileSystem:RootPath"] = rootPath,
|
||||
};
|
||||
config.AddInMemoryCollection(settings!);
|
||||
@@ -65,7 +59,6 @@ public sealed class StatusEndpointTests : IDisposable
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
|
||||
private sealed class StatusResponse
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="EphemeralMongo" Version="3.0.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.TestHost" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" Version="9.10.0" />
|
||||
|
||||
@@ -1,31 +1,36 @@
|
||||
using System;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Attestation.Verification;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Export;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Attestation.Dsse;
|
||||
using StellaOps.Excititor.Attestation.Signing;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
internal static class TestServiceOverrides
|
||||
{
|
||||
public static void Apply(IServiceCollection services)
|
||||
{
|
||||
services.RemoveAll<IVexConnector>();
|
||||
services.RemoveAll<IVexIngestOrchestrator>();
|
||||
services.RemoveAll<IVexConnectorStateRepository>();
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
internal static class TestServiceOverrides
|
||||
{
|
||||
public static void Apply(IServiceCollection services)
|
||||
{
|
||||
services.RemoveAll<IVexConnector>();
|
||||
services.RemoveAll<IVexIngestOrchestrator>();
|
||||
services.RemoveAll<IVexConnectorStateRepository>();
|
||||
services.RemoveAll<IVexRawStore>();
|
||||
services.RemoveAll<IAppendOnlyLinksetStore>();
|
||||
services.RemoveAll<IVexLinksetStore>();
|
||||
services.RemoveAll<IVexObservationStore>();
|
||||
services.RemoveAll<IVexClaimStore>();
|
||||
services.RemoveAll<IVexExportCacheService>();
|
||||
services.RemoveAll<IVexExportDataSource>();
|
||||
services.RemoveAll<IVexExportStore>();
|
||||
@@ -37,110 +42,115 @@ internal static class TestServiceOverrides
|
||||
|
||||
services.AddSingleton<IVexIngestOrchestrator, StubIngestOrchestrator>();
|
||||
services.AddSingleton<IVexConnectorStateRepository, StubConnectorStateRepository>();
|
||||
services.AddSingleton<IVexRawStore, InMemoryVexRawStore>();
|
||||
services.AddSingleton<IAppendOnlyLinksetStore, InMemoryAppendOnlyLinksetStore>();
|
||||
services.AddSingleton<IVexLinksetStore>(sp => (IVexLinksetStore)sp.GetRequiredService<IAppendOnlyLinksetStore>());
|
||||
services.AddSingleton<IVexObservationStore, InMemoryVexObservationStore>();
|
||||
services.AddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
|
||||
services.AddSingleton<IVexExportCacheService, StubExportCacheService>();
|
||||
services.RemoveAll<IExportEngine>();
|
||||
services.AddSingleton<IExportEngine, StubExportEngine>();
|
||||
services.AddSingleton<IVexExportDataSource, StubExportDataSource>();
|
||||
services.RemoveAll<IExportEngine>();
|
||||
services.AddSingleton<IExportEngine, StubExportEngine>();
|
||||
services.AddSingleton<IVexExportDataSource, StubExportDataSource>();
|
||||
services.AddSingleton<IVexExportStore, StubExportStore>();
|
||||
services.AddSingleton<IVexCacheIndex, StubCacheIndex>();
|
||||
services.AddSingleton<IVexCacheMaintenance, StubCacheMaintenance>();
|
||||
services.AddSingleton<IVexAttestationClient, StubAttestationClient>();
|
||||
services.AddSingleton<IVexSigner, StubSigner>();
|
||||
services.AddSingleton<IAirgapImportStore, StubAirgapImportStore>();
|
||||
|
||||
services.RemoveAll<IHostedService>();
|
||||
services.AddSingleton<IHostedService, NoopHostedService>();
|
||||
}
|
||||
|
||||
private sealed class StubExportCacheService : IVexExportCacheService
|
||||
{
|
||||
public ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<int> PruneDanglingAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(0);
|
||||
|
||||
public ValueTask<int> PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(0);
|
||||
}
|
||||
|
||||
private sealed class StubExportEngine : IExportEngine
|
||||
{
|
||||
public ValueTask<VexExportManifest> ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
var manifest = new VexExportManifest(
|
||||
exportId: "stub/export",
|
||||
querySignature: VexQuerySignature.FromQuery(context.Query),
|
||||
format: context.Format,
|
||||
createdAt: DateTimeOffset.UtcNow,
|
||||
artifact: new VexContentAddress("sha256", "stub"),
|
||||
claimCount: 0,
|
||||
sourceProviders: Array.Empty<string>());
|
||||
|
||||
return ValueTask.FromResult(manifest);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubExportDataSource : IVexExportDataSource
|
||||
{
|
||||
public ValueTask<VexExportDataSet> FetchAsync(VexQuery query, CancellationToken cancellationToken)
|
||||
{
|
||||
return ValueTask.FromResult(new VexExportDataSet(
|
||||
ImmutableArray<VexConsensus>.Empty,
|
||||
ImmutableArray<VexClaim>.Empty,
|
||||
ImmutableArray<string>.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubExportStore : IVexExportStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexExportManifest> _store = new();
|
||||
|
||||
public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_store.TryGetValue((signature.Value, format), out var manifest);
|
||||
return ValueTask.FromResult(manifest);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_store[(manifest.QuerySignature.Value, manifest.Format)] = manifest;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubCacheIndex : IVexCacheIndex
|
||||
{
|
||||
private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexCacheEntry> _entries = new();
|
||||
|
||||
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_entries.TryGetValue((signature.Value, format), out var entry);
|
||||
return ValueTask.FromResult(entry);
|
||||
}
|
||||
|
||||
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_entries.TryRemove((signature.Value, format), out _);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_entries[(entry.QuerySignature.Value, entry.Format)] = entry;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubCacheMaintenance : IVexCacheMaintenance
|
||||
{
|
||||
public ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult(0);
|
||||
|
||||
public ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> ValueTask.FromResult(0);
|
||||
}
|
||||
|
||||
|
||||
services.RemoveAll<IHostedService>();
|
||||
services.AddSingleton<IHostedService, NoopHostedService>();
|
||||
}
|
||||
|
||||
private sealed class StubExportCacheService : IVexExportCacheService
|
||||
{
|
||||
public ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<int> PruneDanglingAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(0);
|
||||
|
||||
public ValueTask<int> PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(0);
|
||||
}
|
||||
|
||||
private sealed class StubExportEngine : IExportEngine
|
||||
{
|
||||
public ValueTask<VexExportManifest> ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
var manifest = new VexExportManifest(
|
||||
exportId: "stub/export",
|
||||
querySignature: VexQuerySignature.FromQuery(context.Query),
|
||||
format: context.Format,
|
||||
createdAt: DateTimeOffset.UtcNow,
|
||||
artifact: new VexContentAddress("sha256", "stub"),
|
||||
claimCount: 0,
|
||||
sourceProviders: Array.Empty<string>());
|
||||
|
||||
return ValueTask.FromResult(manifest);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubExportDataSource : IVexExportDataSource
|
||||
{
|
||||
public ValueTask<VexExportDataSet> FetchAsync(VexQuery query, CancellationToken cancellationToken)
|
||||
{
|
||||
return ValueTask.FromResult(new VexExportDataSet(
|
||||
ImmutableArray<VexConsensus>.Empty,
|
||||
ImmutableArray<VexClaim>.Empty,
|
||||
ImmutableArray<string>.Empty));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubExportStore : IVexExportStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexExportManifest> _store = new();
|
||||
|
||||
public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
{
|
||||
_store.TryGetValue((signature.Value, format), out var manifest);
|
||||
return ValueTask.FromResult(manifest);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
_store[(manifest.QuerySignature.Value, manifest.Format)] = manifest;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubCacheIndex : IVexCacheIndex
|
||||
{
|
||||
private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexCacheEntry> _entries = new();
|
||||
|
||||
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
{
|
||||
_entries.TryGetValue((signature.Value, format), out var entry);
|
||||
return ValueTask.FromResult(entry);
|
||||
}
|
||||
|
||||
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
|
||||
{
|
||||
_entries.TryRemove((signature.Value, format), out _);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
_entries[(entry.QuerySignature.Value, entry.Format)] = entry;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubCacheMaintenance : IVexCacheMaintenance
|
||||
{
|
||||
public ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(0);
|
||||
|
||||
public ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(0);
|
||||
}
|
||||
|
||||
private sealed class StubAttestationClient : IVexAttestationClient
|
||||
{
|
||||
public ValueTask<VexAttestationResponse> SignAsync(VexAttestationRequest request, CancellationToken cancellationToken)
|
||||
@@ -166,31 +176,31 @@ internal static class TestServiceOverrides
|
||||
diagnostics);
|
||||
return ValueTask.FromResult(response);
|
||||
}
|
||||
|
||||
|
||||
public ValueTask<VexAttestationVerification> VerifyAsync(VexAttestationVerificationRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
var verification = new VexAttestationVerification(true, VexAttestationDiagnostics.Empty);
|
||||
return ValueTask.FromResult(verification);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private sealed class StubConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, VexConnectorState> _states = new(StringComparer.Ordinal);
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
_states.TryGetValue(connectorId, out var state);
|
||||
return ValueTask.FromResult(state);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
{
|
||||
_states.TryGetValue(connectorId, out var state);
|
||||
return ValueTask.FromResult(state);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
_states[state.ConnectorId] = state;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
IReadOnlyCollection<VexConnectorState> snapshot = _states.Values.ToList();
|
||||
return ValueTask.FromResult(snapshot);
|
||||
@@ -288,26 +298,26 @@ internal static class TestServiceOverrides
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubIngestOrchestrator : IVexIngestOrchestrator
|
||||
{
|
||||
public Task<InitSummary> InitializeAsync(IngestInitOptions options, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new InitSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray<InitProviderResult>.Empty));
|
||||
|
||||
public Task<IngestRunSummary> RunAsync(IngestRunOptions options, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new IngestRunSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray<ProviderRunResult>.Empty));
|
||||
|
||||
public Task<IngestRunSummary> ResumeAsync(IngestResumeOptions options, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new IngestRunSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray<ProviderRunResult>.Empty));
|
||||
|
||||
public Task<ReconcileSummary> ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new ReconcileSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray<ReconcileProviderResult>.Empty));
|
||||
}
|
||||
|
||||
private sealed class NoopHostedService : IHostedService
|
||||
{
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubIngestOrchestrator : IVexIngestOrchestrator
|
||||
{
|
||||
public Task<InitSummary> InitializeAsync(IngestInitOptions options, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new InitSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray<InitProviderResult>.Empty));
|
||||
|
||||
public Task<IngestRunSummary> RunAsync(IngestRunOptions options, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new IngestRunSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray<ProviderRunResult>.Empty));
|
||||
|
||||
public Task<IngestRunSummary> ResumeAsync(IngestResumeOptions options, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new IngestRunSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray<ProviderRunResult>.Empty));
|
||||
|
||||
public Task<ReconcileSummary> ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new ReconcileSummary(Guid.Empty, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, ImmutableArray<ReconcileProviderResult>.Empty));
|
||||
}
|
||||
|
||||
private sealed class NoopHostedService : IHostedService
|
||||
{
|
||||
public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
@@ -39,9 +38,9 @@ public sealed class TestWebApplicationFactory : WebApplicationFactory<Program>
|
||||
{
|
||||
var defaults = new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = "mongodb://localhost:27017",
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "excititor-tests",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = "test",
|
||||
["Postgres:Excititor:ConnectionString"] = "Host=localhost;Username=postgres;Password=postgres;Database=excititor_tests",
|
||||
["Postgres:Excititor:SchemaName"] = "vex",
|
||||
["Excititor:Storage:DefaultTenant"] = "test",
|
||||
};
|
||||
config.AddInMemoryCollection(defaults);
|
||||
_configureConfiguration?.Invoke(config);
|
||||
|
||||
@@ -2,32 +2,26 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using EphemeralMongo;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using System.Net;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class VexAttestationLinkEndpointTests : IDisposable
|
||||
{
|
||||
private readonly IMongoRunner _runner;
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public VexAttestationLinkEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "vex_attestation_links",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = "tests",
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
@@ -35,52 +29,22 @@ public sealed class VexAttestationLinkEndpointTests : IDisposable
|
||||
TestServiceOverrides.Apply(services);
|
||||
services.AddTestAuthentication();
|
||||
});
|
||||
|
||||
SeedLink();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAttestationLink_ReturnsPayload()
|
||||
public async Task GetAttestationLink_ReturnsServiceUnavailable()
|
||||
{
|
||||
using var client = _factory.CreateClient(new WebApplicationFactoryClientOptions { AllowAutoRedirect = false });
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.read");
|
||||
|
||||
var response = await client.GetAsync("/v1/vex/attestations/att-123");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var payload = await response.Content.ReadFromJsonAsync<VexAttestationPayload>();
|
||||
Assert.NotNull(payload);
|
||||
Assert.Equal("att-123", payload!.AttestationId);
|
||||
Assert.Equal("supplier-a", payload.SupplierId);
|
||||
Assert.Equal("CVE-2025-0001", payload.VulnerabilityId);
|
||||
Assert.Equal("pkg:demo", payload.ProductKey);
|
||||
}
|
||||
|
||||
private void SeedLink()
|
||||
{
|
||||
var client = new MongoDB.Driver.MongoClient(_runner.ConnectionString);
|
||||
var database = client.GetDatabase(vex_attestation_links);
|
||||
var collection = database.GetCollection<VexAttestationLinkRecord>(VexMongoCollectionNames.Attestations);
|
||||
|
||||
var record = new VexAttestationLinkRecord
|
||||
{
|
||||
AttestationId = "att-123",
|
||||
SupplierId = "supplier-a",
|
||||
ObservationId = "obs-1",
|
||||
LinksetId = "link-1",
|
||||
VulnerabilityId = "CVE-2025-0001",
|
||||
ProductKey = "pkg:demo",
|
||||
JustificationSummary = "summary",
|
||||
IssuedAt = DateTime.UtcNow,
|
||||
Metadata = new Dictionary<string, string> { ["policyRevisionId"] = "rev-1" },
|
||||
};
|
||||
|
||||
collection.InsertOne(record);
|
||||
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
|
||||
var payload = await response.Content.ReadAsStringAsync();
|
||||
Assert.Contains("temporarily unavailable", payload, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using Xunit;
|
||||
|
||||
@@ -86,10 +85,10 @@ public sealed class VexEvidenceChunkServiceTests
|
||||
_claims = claims;
|
||||
}
|
||||
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken)
|
||||
{
|
||||
var query = _claims
|
||||
.Where(claim => claim.VulnerabilityId == vulnerabilityId)
|
||||
@@ -102,6 +101,16 @@ public sealed class VexEvidenceChunkServiceTests
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(query.ToList());
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = _claims
|
||||
.Where(claim => claim.VulnerabilityId == vulnerabilityId)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(results);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedTimeProvider : TimeProvider
|
||||
|
||||
@@ -5,35 +5,27 @@ using System.Linq;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using EphemeralMongo;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using System.Net;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class VexEvidenceChunksEndpointTests : IDisposable
|
||||
{
|
||||
private readonly IMongoRunner _runner;
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public VexEvidenceChunksEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "vex_chunks_tests",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = "tests",
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
@@ -41,37 +33,24 @@ public sealed class VexEvidenceChunksEndpointTests : IDisposable
|
||||
TestServiceOverrides.Apply(services);
|
||||
services.AddTestAuthentication();
|
||||
});
|
||||
|
||||
SeedStatements();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChunksEndpoint_Filters_ByProvider_AndStreamsNdjson()
|
||||
public async Task ChunksEndpoint_ReturnsServiceUnavailable_DuringMigration()
|
||||
{
|
||||
using var client = _factory.CreateClient(new WebApplicationFactoryClientOptions { AllowAutoRedirect = false });
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.read");
|
||||
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "tests");
|
||||
|
||||
var response = await client.GetAsync("/v1/vex/evidence/chunks?vulnerabilityId=CVE-2025-0001&productKey=pkg:docker/demo&providerId=provider-b&limit=1");
|
||||
response.EnsureSuccessStatusCode();
|
||||
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
|
||||
|
||||
Assert.True(response.Headers.TryGetValues("Excititor-Results-Truncated", out var truncatedValues));
|
||||
Assert.Contains("true", truncatedValues, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
var lines = body.Split(n, StringSplitOptions.RemoveEmptyEntries);
|
||||
Assert.Single(lines);
|
||||
|
||||
var chunk = JsonSerializer.Deserialize<VexEvidenceChunkResponse>(lines[0], new JsonSerializerOptions(JsonSerializerDefaults.Web));
|
||||
Assert.NotNull(chunk);
|
||||
Assert.Equal("provider-b", chunk!.ProviderId);
|
||||
Assert.Equal("NotAffected", chunk.Status);
|
||||
Assert.Equal("pkg:docker/demo", chunk.Scope.Key);
|
||||
Assert.Equal("CVE-2025-0001", chunk.VulnerabilityId);
|
||||
var problem = await response.Content.ReadAsStringAsync();
|
||||
Assert.Contains("temporarily unavailable", problem, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChunksEndpoint_Sets_Results_Headers()
|
||||
public async Task ChunksEndpoint_ReportsMigrationStatusHeaders()
|
||||
{
|
||||
using var client = _factory.CreateClient(new WebApplicationFactoryClientOptions { AllowAutoRedirect = false });
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "vex.read");
|
||||
@@ -79,70 +58,13 @@ public sealed class VexEvidenceChunksEndpointTests : IDisposable
|
||||
|
||||
// No provider filter; limit forces truncation so headers should reflect total > limit.
|
||||
var response = await client.GetAsync("/v1/vex/evidence/chunks?vulnerabilityId=CVE-2025-0001&productKey=pkg:docker/demo&limit=1");
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
Assert.Equal("application/x-ndjson", response.Content.Headers.ContentType?.MediaType);
|
||||
|
||||
Assert.True(response.Headers.TryGetValues("Excititor-Results-Total", out var totalValues));
|
||||
Assert.Equal("3", totalValues.Single());
|
||||
|
||||
Assert.True(response.Headers.TryGetValues("Excititor-Results-Truncated", out var truncatedValues));
|
||||
Assert.Equal("true", truncatedValues.Single(), ignoreCase: true);
|
||||
}
|
||||
|
||||
private void SeedStatements()
|
||||
{
|
||||
var client = new MongoClient(_runner.ConnectionString);
|
||||
var database = client.GetDatabase("vex_chunks_tests");
|
||||
var collection = database.GetCollection<VexStatementRecord>(VexMongoCollectionNames.Statements);
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var claims = new[]
|
||||
{
|
||||
CreateClaim("provider-a", VexClaimStatus.Affected, now.AddHours(-6), now.AddHours(-5), 0.9),
|
||||
CreateClaim("provider-b", VexClaimStatus.NotAffected, now.AddHours(-4), now.AddHours(-3), 0.2),
|
||||
CreateClaim("provider-c", VexClaimStatus.Affected, now.AddHours(-2), now.AddHours(-1), 0.5)
|
||||
};
|
||||
|
||||
var records = claims
|
||||
.Select(claim => VexStatementRecord.FromDomain(claim, now))
|
||||
.ToList();
|
||||
|
||||
collection.InsertMany(records);
|
||||
}
|
||||
|
||||
private static VexClaim CreateClaim(string providerId, VexClaimStatus status, DateTimeOffset firstSeen, DateTimeOffset lastSeen, double? score)
|
||||
{
|
||||
var product = new VexProduct("pkg:docker/demo", "demo", "1.0.0", "pkg:docker/demo:1.0.0", null, new[] { "component-a" });
|
||||
var document = new VexClaimDocument(
|
||||
VexDocumentFormat.SbomCycloneDx,
|
||||
digest: Guid.NewGuid().ToString("N"),
|
||||
sourceUri: new Uri("https://example.test/vex.json"),
|
||||
revision: "r1",
|
||||
signature: new VexSignatureMetadata("cosign", "demo", "issuer", keyId: "kid", verifiedAt: firstSeen, transparencyLogReference: null));
|
||||
|
||||
var signals = score.HasValue
|
||||
? new VexSignalSnapshot(new VexSeveritySignal("cvss", score, "low", vector: null), kev: null, epss: null)
|
||||
: null;
|
||||
|
||||
return new VexClaim(
|
||||
"CVE-2025-0001",
|
||||
providerId,
|
||||
product,
|
||||
status,
|
||||
document,
|
||||
firstSeen,
|
||||
lastSeen,
|
||||
justification: VexJustification.ComponentNotPresent,
|
||||
detail: "demo detail",
|
||||
confidence: null,
|
||||
signals: signals,
|
||||
additionalMetadata: null);
|
||||
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
|
||||
var detail = await response.Content.ReadAsStringAsync();
|
||||
Assert.Contains("temporarily unavailable", detail, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Net.Http.Json;
|
||||
using EphemeralMongo;
|
||||
using System.Threading;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using Xunit;
|
||||
|
||||
@@ -15,21 +17,16 @@ namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class VexLinksetListEndpointTests : IDisposable
|
||||
{
|
||||
private readonly IMongoRunner _runner;
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public VexLinksetListEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "linksets_tests",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = "tests",
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
@@ -56,7 +53,8 @@ public sealed class VexLinksetListEndpointTests : IDisposable
|
||||
Assert.Single(payload!.Items);
|
||||
|
||||
var item = payload.Items.Single();
|
||||
Assert.Equal("CVE-2025-0001:pkg:demo/app", item.LinksetId);
|
||||
var expectedId = VexLinkset.CreateLinksetId("tests", "CVE-2025-0001", "pkg:demo/app");
|
||||
Assert.Equal(expectedId, item.LinksetId);
|
||||
Assert.Equal("CVE-2025-0001", item.VulnerabilityId);
|
||||
Assert.Equal("pkg:demo/app", item.ProductKey);
|
||||
|
||||
@@ -69,72 +67,34 @@ public sealed class VexLinksetListEndpointTests : IDisposable
|
||||
|
||||
private void SeedObservations()
|
||||
{
|
||||
var client = new MongoClient(_runner.ConnectionString);
|
||||
var database = client.GetDatabase("linksets_tests");
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Observations);
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var store = scope.ServiceProvider.GetRequiredService<IAppendOnlyLinksetStore>();
|
||||
|
||||
var observations = new List<BsonDocument>
|
||||
var scopeMetadata = new VexProductScope(
|
||||
key: "pkg:demo/app",
|
||||
name: "demo app",
|
||||
version: null,
|
||||
purl: "pkg:demo/app",
|
||||
cpe: null,
|
||||
componentIdentifiers: Array.Empty<string>());
|
||||
|
||||
var observations = new[]
|
||||
{
|
||||
new()
|
||||
{
|
||||
{ "_id", "obs-1" },
|
||||
{ "Tenant", "tests" },
|
||||
{ "ObservationId", "obs-1" },
|
||||
{ "VulnerabilityId", "cve-2025-0001" },
|
||||
{ "ProductKey", "pkg:demo/app" },
|
||||
{ "ProviderId", "provider-a" },
|
||||
{ "Status", "affected" },
|
||||
{ "StreamId", "stream" },
|
||||
{ "CreatedAt", DateTime.UtcNow },
|
||||
{ "Document", new BsonDocument { { "Digest", "digest-1" }, { "Format", "csaf" }, { "SourceUri", "https://example.test/a.json" } } },
|
||||
{ "Statements", new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "VulnerabilityId", "cve-2025-0001" },
|
||||
{ "ProductKey", "pkg:demo/app" },
|
||||
{ "Status", "affected" },
|
||||
{ "LastObserved", DateTime.UtcNow },
|
||||
{ "Purl", "pkg:demo/app" }
|
||||
}
|
||||
}
|
||||
},
|
||||
{ "Linkset", new BsonDocument { { "Purls", new BsonArray { "pkg:demo/app" } } } }
|
||||
},
|
||||
new()
|
||||
{
|
||||
{ "_id", "obs-2" },
|
||||
{ "Tenant", "tests" },
|
||||
{ "ObservationId", "obs-2" },
|
||||
{ "VulnerabilityId", "cve-2025-0001" },
|
||||
{ "ProductKey", "pkg:demo/app" },
|
||||
{ "ProviderId", "provider-b" },
|
||||
{ "Status", "fixed" },
|
||||
{ "StreamId", "stream" },
|
||||
{ "CreatedAt", DateTime.UtcNow.AddMinutes(1) },
|
||||
{ "Document", new BsonDocument { { "Digest", "digest-2" }, { "Format", "csaf" }, { "SourceUri", "https://example.test/b.json" } } },
|
||||
{ "Statements", new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "VulnerabilityId", "cve-2025-0001" },
|
||||
{ "ProductKey", "pkg:demo/app" },
|
||||
{ "Status", "fixed" },
|
||||
{ "LastObserved", DateTime.UtcNow },
|
||||
{ "Purl", "pkg:demo/app" }
|
||||
}
|
||||
}
|
||||
},
|
||||
{ "Linkset", new BsonDocument { { "Purls", new BsonArray { "pkg:demo/app" } } } }
|
||||
}
|
||||
new VexLinksetObservationRefModel("obs-1", "provider-a", "affected", 0.8),
|
||||
new VexLinksetObservationRefModel("obs-2", "provider-b", "fixed", 0.9),
|
||||
};
|
||||
|
||||
collection.InsertMany(observations);
|
||||
store.AppendObservationsBatchAsync(
|
||||
tenant: "tests",
|
||||
vulnerabilityId: "CVE-2025-0001",
|
||||
productKey: "pkg:demo/app",
|
||||
observations: observations,
|
||||
scope: scopeMetadata,
|
||||
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Net.Http.Json;
|
||||
using EphemeralMongo;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Threading;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using Xunit;
|
||||
|
||||
@@ -15,21 +17,16 @@ namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class VexObservationListEndpointTests : IDisposable
|
||||
{
|
||||
private readonly IMongoRunner _runner;
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public VexObservationListEndpointTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "observations_tests",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = "tests",
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
@@ -66,45 +63,55 @@ public sealed class VexObservationListEndpointTests : IDisposable
|
||||
|
||||
private void SeedObservation()
|
||||
{
|
||||
var client = new MongoClient(_runner.ConnectionString);
|
||||
var database = client.GetDatabase("observations_tests");
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Observations);
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var store = scope.ServiceProvider.GetRequiredService<IVexObservationStore>();
|
||||
|
||||
var record = new BsonDocument
|
||||
{
|
||||
{ "_id", "obs-1" },
|
||||
{ "Tenant", "tests" },
|
||||
{ "ObservationId", "obs-1" },
|
||||
{ "VulnerabilityId", "cve-2025-0001" },
|
||||
{ "ProductKey", "pkg:demo/app" },
|
||||
{ "ProviderId", "provider-a" },
|
||||
{ "Status", "affected" },
|
||||
{ "StreamId", "stream" },
|
||||
{ "CreatedAt", DateTime.UtcNow },
|
||||
{ "Document", new BsonDocument { { "Digest", "digest-1" }, { "Format", "csaf" }, { "SourceUri", "https://example.test/vex.json" } } },
|
||||
{ "Upstream", new BsonDocument { { "UpstreamId", "up-1" }, { "ContentHash", "sha256:digest-1" }, { "Signature", new BsonDocument { { "Present", true }, { "Subject", "sub" }, { "Issuer", "iss" }, { "VerifiedAt", DateTime.UtcNow } } } } },
|
||||
{ "Content", new BsonDocument { { "Format", "csaf" }, { "Raw", new BsonDocument { { "document", "payload" } } } } },
|
||||
{ "Statements", new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "VulnerabilityId", "cve-2025-0001" },
|
||||
{ "ProductKey", "pkg:demo/app" },
|
||||
{ "Status", "affected" },
|
||||
{ "LastObserved", DateTime.UtcNow },
|
||||
{ "Purl", "pkg:demo/app" }
|
||||
}
|
||||
}
|
||||
},
|
||||
{ "Linkset", new BsonDocument { { "Purls", new BsonArray { "pkg:demo/app" } } } }
|
||||
};
|
||||
var now = DateTimeOffset.Parse("2025-12-01T00:00:00Z");
|
||||
var observation = new VexObservation(
|
||||
observationId: "obs-1",
|
||||
tenant: "tests",
|
||||
providerId: "provider-a",
|
||||
streamId: "stream",
|
||||
upstream: new VexObservationUpstream(
|
||||
upstreamId: "up-1",
|
||||
documentVersion: "1",
|
||||
fetchedAt: now,
|
||||
receivedAt: now,
|
||||
contentHash: "sha256:digest-1",
|
||||
signature: new VexObservationSignature(
|
||||
present: true,
|
||||
format: "dsse",
|
||||
keyId: "key-1",
|
||||
signature: "stub-signature")),
|
||||
statements: ImmutableArray.Create(new VexObservationStatement(
|
||||
vulnerabilityId: "cve-2025-0001",
|
||||
productKey: "pkg:demo/app",
|
||||
status: VexClaimStatus.Affected,
|
||||
lastObserved: now,
|
||||
locator: null,
|
||||
justification: null,
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
purl: "pkg:demo/app",
|
||||
cpe: null,
|
||||
evidence: null,
|
||||
metadata: null)),
|
||||
content: new VexObservationContent(
|
||||
format: "csaf",
|
||||
specVersion: "2.0",
|
||||
raw: JsonNode.Parse("{\"document\":\"payload\"}")!),
|
||||
linkset: new VexObservationLinkset(
|
||||
aliases: new[] { "cve-2025-0001" },
|
||||
purls: new[] { "pkg:demo/app" },
|
||||
cpes: Array.Empty<string>(),
|
||||
references: Array.Empty<VexObservationReference>()),
|
||||
createdAt: now);
|
||||
|
||||
collection.InsertOne(record);
|
||||
store.InsertAsync(observation, CancellationToken.None).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,9 +6,7 @@ using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Services;
|
||||
using Xunit;
|
||||
|
||||
@@ -119,10 +117,10 @@ public sealed class VexObservationProjectionServiceTests
|
||||
_claims = claims;
|
||||
}
|
||||
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken)
|
||||
{
|
||||
var query = _claims
|
||||
.Where(claim => string.Equals(claim.VulnerabilityId, vulnerabilityId, StringComparison.OrdinalIgnoreCase))
|
||||
@@ -135,6 +133,16 @@ public sealed class VexObservationProjectionServiceTests
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(query.ToList());
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = _claims
|
||||
.Where(claim => string.Equals(claim.VulnerabilityId, vulnerabilityId, StringComparison.OrdinalIgnoreCase))
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(results);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedTimeProvider : TimeProvider
|
||||
|
||||
@@ -3,34 +3,26 @@ using System.Collections.Generic;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using EphemeralMongo;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.WebService.Contracts;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.WebService.Tests;
|
||||
|
||||
public sealed class VexRawEndpointsTests : IDisposable
|
||||
public sealed class VexRawEndpointsTests
|
||||
{
|
||||
private readonly IMongoRunner _runner;
|
||||
private readonly TestWebApplicationFactory _factory;
|
||||
|
||||
public VexRawEndpointsTests()
|
||||
{
|
||||
_runner = MongoRunner.Run(new MongoRunnerOptions { UseSingleNodeReplicaSet = true });
|
||||
|
||||
_factory = new TestWebApplicationFactory(
|
||||
configureConfiguration: configuration =>
|
||||
{
|
||||
configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Excititor:Storage:Mongo:ConnectionString"] = _runner.ConnectionString,
|
||||
["Excititor:Storage:Mongo:DatabaseName"] = "vex_raw_tests",
|
||||
["Excititor:Storage:Mongo:DefaultTenant"] = "tests",
|
||||
["Excititor:Storage:DefaultTenant"] = "tests",
|
||||
});
|
||||
},
|
||||
configureServices: services =>
|
||||
@@ -99,9 +91,4 @@ public sealed class VexRawEndpointsTests : IDisposable
|
||||
});
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_factory.Dispose();
|
||||
_runner.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,413 +1,418 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Mongo2Go;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Aoc;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
using StellaOps.Excititor.Worker.Orchestration;
|
||||
using StellaOps.Excititor.Worker.Scheduling;
|
||||
using StellaOps.Excititor.Worker.Signature;
|
||||
using StellaOps.Plugin;
|
||||
using Xunit;
|
||||
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Tests;
|
||||
|
||||
public sealed class DefaultVexProviderRunnerIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private readonly MongoDbRunner _runner;
|
||||
private readonly MongoClient _client;
|
||||
|
||||
public DefaultVexProviderRunnerIntegrationTests()
|
||||
{
|
||||
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
_client = new MongoClient(_runner.ConnectionString);
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_LargeBatch_IdempotentAcrossRestart()
|
||||
{
|
||||
var specs = CreateDocumentSpecs(count: 48);
|
||||
var databaseName = $"vex-worker-batch-{Guid.NewGuid():N}";
|
||||
var (provider, guard, database, connector) = ConfigureIntegrationServices(databaseName, specs);
|
||||
|
||||
try
|
||||
{
|
||||
var time = new FixedTimeProvider(new DateTimeOffset(2025, 10, 28, 8, 0, 0, TimeSpan.Zero));
|
||||
var runner = CreateRunner(provider, time);
|
||||
var schedule = new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(10), TimeSpan.Zero, VexConnectorSettings.Empty);
|
||||
|
||||
await runner.RunAsync(schedule, CancellationToken.None);
|
||||
|
||||
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var stored = await rawCollection.Find(FilterDefinition<BsonDocument>.Empty).ToListAsync();
|
||||
stored.Should().HaveCount(specs.Count);
|
||||
|
||||
// Supersedes metadata is preserved for chained documents.
|
||||
var target = specs[17];
|
||||
var storedTarget = stored.Single(doc => doc["_id"] == target.Digest);
|
||||
storedTarget["Metadata"].AsBsonDocument.TryGetValue("aoc.supersedes", out var supersedesValue)
|
||||
.Should().BeTrue();
|
||||
supersedesValue!.AsString.Should().Be(target.Metadata["aoc.supersedes"]);
|
||||
|
||||
await runner.RunAsync(schedule, CancellationToken.None);
|
||||
|
||||
var afterRestart = await rawCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
|
||||
afterRestart.Should().Be(specs.Count);
|
||||
|
||||
// Guard invoked for every document across both runs.
|
||||
guard.Invocations
|
||||
.GroupBy(doc => doc.Upstream.ContentHash)
|
||||
.Should().OnlyContain(group => group.Count() == 2);
|
||||
|
||||
// Verify provenance still carries supersedes linkage.
|
||||
var provenance = guard.Invocations
|
||||
.Where(doc => doc.Upstream.ContentHash == target.Digest)
|
||||
.Select(doc => doc.Upstream.Provenance["aoc.supersedes"])
|
||||
.ToImmutableArray();
|
||||
provenance.Should().HaveCount(2).And.AllBeEquivalentTo(target.Metadata["aoc.supersedes"]);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await _client.DropDatabaseAsync(databaseName);
|
||||
await provider.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_WhenGuardFails_RestartCompletesSuccessfully()
|
||||
{
|
||||
var specs = CreateDocumentSpecs(count: 24);
|
||||
var failureDigest = specs[9].Digest;
|
||||
var databaseName = $"vex-worker-guard-{Guid.NewGuid():N}";
|
||||
var (provider, guard, database, connector) = ConfigureIntegrationServices(databaseName, specs, failureDigest);
|
||||
|
||||
try
|
||||
{
|
||||
var time = new FixedTimeProvider(new DateTimeOffset(2025, 10, 28, 9, 0, 0, TimeSpan.Zero));
|
||||
var runner = CreateRunner(provider, time);
|
||||
var schedule = new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(5), TimeSpan.Zero, VexConnectorSettings.Empty);
|
||||
|
||||
await Assert.ThrowsAsync<ExcititorAocGuardException>(() => runner.RunAsync(schedule, CancellationToken.None).AsTask());
|
||||
|
||||
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var storedCount = await rawCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
|
||||
storedCount.Should().Be(9); // documents before the failing digest persist
|
||||
|
||||
guard.FailDigest = null;
|
||||
// Advance past the quarantine duration (30 mins) since AOC guard failures are non-retryable
|
||||
time.Advance(TimeSpan.FromMinutes(35));
|
||||
await runner.RunAsync(schedule, CancellationToken.None);
|
||||
|
||||
var finalCount = await rawCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
|
||||
finalCount.Should().Be(specs.Count);
|
||||
|
||||
guard.Invocations.Count(doc => doc.Upstream.ContentHash == failureDigest).Should().Be(2);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await _client.DropDatabaseAsync(databaseName);
|
||||
await provider.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private (ServiceProvider Provider, RecordingVexRawWriteGuard Guard, IMongoDatabase Database, BatchingConnector Connector) ConfigureIntegrationServices(
|
||||
string databaseName,
|
||||
IReadOnlyList<DocumentSpec> specs,
|
||||
string? guardFailureDigest = null)
|
||||
{
|
||||
var database = _client.GetDatabase(databaseName);
|
||||
var optionsValue = new VexMongoStorageOptions
|
||||
{
|
||||
ConnectionString = _runner.ConnectionString,
|
||||
DatabaseName = databaseName,
|
||||
DefaultTenant = "tenant-integration",
|
||||
GridFsInlineThresholdBytes = 64 * 1024,
|
||||
};
|
||||
var options = Microsoft.Extensions.Options.Options.Create(optionsValue);
|
||||
var sessionProvider = new DirectSessionProvider(_client);
|
||||
var guard = new RecordingVexRawWriteGuard { FailDigest = guardFailureDigest };
|
||||
var rawStore = new MongoVexRawStore(_client, database, options, sessionProvider, guard);
|
||||
var providerStore = new MongoVexProviderStore(database);
|
||||
var stateRepository = new MongoVexConnectorStateRepository(database);
|
||||
var connector = new BatchingConnector("integration:test", specs);
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddSingleton<IVexConnector>(connector);
|
||||
services.AddSingleton<IVexRawStore>(rawStore);
|
||||
services.AddSingleton<IVexProviderStore>(providerStore);
|
||||
services.AddSingleton<IVexConnectorStateRepository>(stateRepository);
|
||||
services.AddSingleton<IVexClaimStore>(new NoopClaimStore());
|
||||
services.AddSingleton<IVexNormalizerRouter>(new NoopNormalizerRouter());
|
||||
services.AddSingleton<IVexSignatureVerifier>(new NoopSignatureVerifier());
|
||||
|
||||
return (services.BuildServiceProvider(), guard, database, connector);
|
||||
}
|
||||
|
||||
private static DefaultVexProviderRunner CreateRunner(IServiceProvider services, TimeProvider timeProvider)
|
||||
{
|
||||
var options = new VexWorkerOptions
|
||||
{
|
||||
Retry =
|
||||
{
|
||||
BaseDelay = TimeSpan.FromSeconds(5),
|
||||
MaxDelay = TimeSpan.FromMinutes(1),
|
||||
JitterRatio = 0.1,
|
||||
FailureThreshold = 3,
|
||||
QuarantineDuration = TimeSpan.FromMinutes(30),
|
||||
},
|
||||
};
|
||||
|
||||
var orchestratorOptions = Microsoft.Extensions.Options.Options.Create(new VexWorkerOrchestratorOptions { Enabled = false });
|
||||
var orchestratorClient = new NoopOrchestratorClient();
|
||||
var heartbeatService = new VexWorkerHeartbeatService(
|
||||
orchestratorClient,
|
||||
orchestratorOptions,
|
||||
timeProvider,
|
||||
NullLogger<VexWorkerHeartbeatService>.Instance);
|
||||
|
||||
return new DefaultVexProviderRunner(
|
||||
services,
|
||||
new PluginCatalog(),
|
||||
orchestratorClient,
|
||||
heartbeatService,
|
||||
NullLogger<DefaultVexProviderRunner>.Instance,
|
||||
timeProvider,
|
||||
Microsoft.Extensions.Options.Options.Create(options),
|
||||
orchestratorOptions);
|
||||
}
|
||||
|
||||
private static List<DocumentSpec> CreateDocumentSpecs(int count)
|
||||
{
|
||||
var specs = new List<DocumentSpec>(capacity: count);
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var payload = JsonSerializer.Serialize(new
|
||||
{
|
||||
id = i,
|
||||
title = $"VEX advisory {i}",
|
||||
supersedes = i == 0 ? null : $"sha256:batch-{i - 1:D4}",
|
||||
});
|
||||
|
||||
var digest = ComputeDigest(payload);
|
||||
var metadataBuilder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
metadataBuilder["source.vendor"] = "integration-vendor";
|
||||
metadataBuilder["source.connector"] = "integration-connector";
|
||||
metadataBuilder["aoc.supersedes"] = i == 0 ? string.Empty : $"sha256:batch-{i - 1:D4}";
|
||||
|
||||
specs.Add(new DocumentSpec(
|
||||
ProviderId: "integration-provider",
|
||||
Format: VexDocumentFormat.Csaf,
|
||||
SourceUri: new Uri($"https://example.org/vex/{i}.json"),
|
||||
RetrievedAt: new DateTimeOffset(2025, 10, 28, 7, 0, 0, TimeSpan.Zero).AddMinutes(i),
|
||||
Digest: digest,
|
||||
Payload: payload,
|
||||
Metadata: metadataBuilder.ToImmutable()));
|
||||
}
|
||||
|
||||
return specs;
|
||||
}
|
||||
|
||||
private static string ComputeDigest(string payload)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(payload);
|
||||
Span<byte> buffer = stackalloc byte[32];
|
||||
if (SHA256.TryHashData(bytes, buffer, out _))
|
||||
{
|
||||
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
|
||||
}
|
||||
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private sealed record DocumentSpec(
|
||||
string ProviderId,
|
||||
VexDocumentFormat Format,
|
||||
Uri SourceUri,
|
||||
DateTimeOffset RetrievedAt,
|
||||
string Digest,
|
||||
string Payload,
|
||||
ImmutableDictionary<string, string> Metadata)
|
||||
{
|
||||
public VexRawDocument CreateDocument()
|
||||
{
|
||||
var content = Encoding.UTF8.GetBytes(Payload);
|
||||
return new VexRawDocument(
|
||||
ProviderId,
|
||||
Format,
|
||||
SourceUri,
|
||||
RetrievedAt,
|
||||
Digest,
|
||||
new ReadOnlyMemory<byte>(content),
|
||||
Metadata);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class BatchingConnector : IVexConnector
|
||||
{
|
||||
private readonly IReadOnlyList<DocumentSpec> _specs;
|
||||
|
||||
public BatchingConnector(string id, IReadOnlyList<DocumentSpec> specs)
|
||||
{
|
||||
Id = id;
|
||||
_specs = specs;
|
||||
}
|
||||
|
||||
public string Id { get; }
|
||||
|
||||
public IReadOnlyList<DocumentSpec> Specs => _specs;
|
||||
|
||||
public VexProviderKind Kind => VexProviderKind.Vendor;
|
||||
|
||||
public ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public async IAsyncEnumerable<VexRawDocument> FetchAsync(
|
||||
VexConnectorContext context,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var spec in _specs)
|
||||
{
|
||||
var document = spec.CreateDocument();
|
||||
await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
yield return document;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
|
||||
private sealed class RecordingVexRawWriteGuard : IVexRawWriteGuard
|
||||
{
|
||||
private readonly List<RawVexDocumentModel> _invocations = new();
|
||||
|
||||
public IReadOnlyList<RawVexDocumentModel> Invocations => _invocations;
|
||||
|
||||
public string? FailDigest { get; set; }
|
||||
|
||||
public void EnsureValid(RawVexDocumentModel document)
|
||||
{
|
||||
_invocations.Add(document);
|
||||
if (FailDigest is not null && string.Equals(document.Upstream.ContentHash, FailDigest, StringComparison.Ordinal))
|
||||
{
|
||||
var violation = AocViolation.Create(
|
||||
AocViolationCode.SignatureInvalid,
|
||||
"/upstream/digest",
|
||||
"Synthetic guard failure.");
|
||||
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopClaimStore : IVexClaimStore
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Aoc;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Core.Storage;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
using StellaOps.Excititor.Worker.Orchestration;
|
||||
using StellaOps.Excititor.Worker.Scheduling;
|
||||
using StellaOps.Excititor.Worker.Signature;
|
||||
using StellaOps.Plugin;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Excititor.Worker.Tests;
|
||||
|
||||
public sealed class DefaultVexProviderRunnerIntegrationTests
|
||||
{
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_LargeBatch_IdempotentAcrossRestart()
|
||||
{
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
var specs = CreateDocumentSpecs(count: 48);
|
||||
var databaseName = $"vex-worker-batch-{Guid.NewGuid():N}";
|
||||
var (provider, rawStore, connector) = ConfigureIntegrationServices(databaseName, specs);
|
||||
|
||||
try
|
||||
{
|
||||
var time = new FixedTimeProvider(new DateTimeOffset(2025, 10, 28, 8, 0, 0, TimeSpan.Zero));
|
||||
var runner = CreateRunner(provider, time);
|
||||
var schedule = new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(10), TimeSpan.Zero, VexConnectorSettings.Empty);
|
||||
|
||||
await runner.RunAsync(schedule, CancellationToken.None);
|
||||
|
||||
var storedPage = await rawStore.QueryAsync(
|
||||
new VexRawQuery(
|
||||
Tenant: "tenant-integration",
|
||||
ProviderIds: Array.Empty<string>(),
|
||||
Digests: Array.Empty<string>(),
|
||||
Formats: Array.Empty<VexDocumentFormat>(),
|
||||
Since: null,
|
||||
Until: null,
|
||||
Cursor: null,
|
||||
Limit: specs.Count + 10),
|
||||
CancellationToken.None);
|
||||
storedPage.Items.Should().HaveCount(specs.Count);
|
||||
|
||||
// Supersedes metadata is preserved for chained documents.
|
||||
var target = specs[17];
|
||||
var storedTarget = await rawStore.FindByDigestAsync(target.Digest, CancellationToken.None);
|
||||
storedTarget.Should().NotBeNull();
|
||||
storedTarget!.Metadata.TryGetValue("aoc.supersedes", out var supersedesValue)
|
||||
.Should().BeTrue();
|
||||
supersedesValue.Should().Be(target.Metadata["aoc.supersedes"]);
|
||||
|
||||
await runner.RunAsync(schedule, CancellationToken.None);
|
||||
|
||||
var afterRestart = await rawStore.QueryAsync(
|
||||
new VexRawQuery(
|
||||
Tenant: "tenant-integration",
|
||||
ProviderIds: Array.Empty<string>(),
|
||||
Digests: Array.Empty<string>(),
|
||||
Formats: Array.Empty<VexDocumentFormat>(),
|
||||
Since: null,
|
||||
Until: null,
|
||||
Cursor: null,
|
||||
Limit: specs.Count + 10),
|
||||
CancellationToken.None);
|
||||
afterRestart.Items.Should().HaveCount(specs.Count);
|
||||
|
||||
// Guard invoked for every document across both runs.
|
||||
rawStore.Invocations
|
||||
.GroupBy(doc => doc.Digest)
|
||||
.Should().OnlyContain(group => group.Count() == 2);
|
||||
|
||||
// Verify provenance still carries supersedes linkage.
|
||||
var provenance = rawStore.Invocations
|
||||
.Where(doc => string.Equals(doc.Digest, target.Digest, StringComparison.OrdinalIgnoreCase))
|
||||
.Select(doc => doc.Metadata["aoc.supersedes"])
|
||||
.ToImmutableArray();
|
||||
provenance.Should().HaveCount(2).And.AllBeEquivalentTo(target.Metadata["aoc.supersedes"]);
|
||||
}
|
||||
finally
|
||||
{
|
||||
provider.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_WhenGuardFails_RestartCompletesSuccessfully()
|
||||
{
|
||||
var specs = CreateDocumentSpecs(count: 24);
|
||||
var failureDigest = specs[9].Digest;
|
||||
var databaseName = $"vex-worker-guard-{Guid.NewGuid():N}";
|
||||
var (provider, rawStore, connector) = ConfigureIntegrationServices(databaseName, specs, failureDigest);
|
||||
|
||||
try
|
||||
{
|
||||
var time = new FixedTimeProvider(new DateTimeOffset(2025, 10, 28, 9, 0, 0, TimeSpan.Zero));
|
||||
var runner = CreateRunner(provider, time);
|
||||
var schedule = new VexWorkerSchedule(connector.Id, TimeSpan.FromMinutes(5), TimeSpan.Zero, VexConnectorSettings.Empty);
|
||||
|
||||
await Assert.ThrowsAsync<ExcititorAocGuardException>(() => runner.RunAsync(schedule, CancellationToken.None).AsTask());
|
||||
|
||||
var storedCount = (await rawStore.QueryAsync(
|
||||
new VexRawQuery(
|
||||
Tenant: "tenant-integration",
|
||||
ProviderIds: Array.Empty<string>(),
|
||||
Digests: Array.Empty<string>(),
|
||||
Formats: Array.Empty<VexDocumentFormat>(),
|
||||
Since: null,
|
||||
Until: null,
|
||||
Cursor: null,
|
||||
Limit: specs.Count + 10),
|
||||
CancellationToken.None)).Items.Count;
|
||||
storedCount.Should().Be(9); // documents before the failing digest persist
|
||||
|
||||
rawStore.FailDigest = null;
|
||||
// Advance past the quarantine duration (30 mins) since AOC guard failures are non-retryable
|
||||
time.Advance(TimeSpan.FromMinutes(35));
|
||||
await runner.RunAsync(schedule, CancellationToken.None);
|
||||
|
||||
var finalCount = (await rawStore.QueryAsync(
|
||||
new VexRawQuery(
|
||||
Tenant: "tenant-integration",
|
||||
ProviderIds: Array.Empty<string>(),
|
||||
Digests: Array.Empty<string>(),
|
||||
Formats: Array.Empty<VexDocumentFormat>(),
|
||||
Since: null,
|
||||
Until: null,
|
||||
Cursor: null,
|
||||
Limit: specs.Count + 10),
|
||||
CancellationToken.None)).Items.Count;
|
||||
finalCount.Should().Be(specs.Count);
|
||||
|
||||
rawStore.Invocations.Count(doc => string.Equals(doc.Digest, failureDigest, StringComparison.OrdinalIgnoreCase)).Should().Be(2);
|
||||
}
|
||||
finally
|
||||
{
|
||||
provider.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private (ServiceProvider Provider, RecordingRawStore RawStore, BatchingConnector Connector) ConfigureIntegrationServices(
|
||||
string _,
|
||||
IReadOnlyList<DocumentSpec> specs,
|
||||
string? guardFailureDigest = null)
|
||||
{
|
||||
var rawStore = new InMemoryVexRawStore(inlineThresholdBytes: 64 * 1024);
|
||||
var recordingStore = new RecordingRawStore(rawStore)
|
||||
{
|
||||
FailDigest = guardFailureDigest
|
||||
};
|
||||
var providerStore = new InMemoryVexProviderStore();
|
||||
var stateRepository = new InMemoryVexConnectorStateRepository();
|
||||
var connector = new BatchingConnector("integration:test", specs);
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddSingleton<IVexConnector>(connector);
|
||||
services.AddSingleton<IVexRawStore>(recordingStore);
|
||||
services.AddSingleton(recordingStore);
|
||||
services.AddSingleton<IVexProviderStore>(providerStore);
|
||||
services.AddSingleton<IVexConnectorStateRepository>(stateRepository);
|
||||
services.AddSingleton<IVexClaimStore>(new NoopClaimStore());
|
||||
services.AddSingleton<IVexNormalizerRouter>(new NoopNormalizerRouter());
|
||||
services.AddSingleton<IVexSignatureVerifier>(new NoopSignatureVerifier());
|
||||
|
||||
return (services.BuildServiceProvider(), recordingStore, connector);
|
||||
}
|
||||
|
||||
private static DefaultVexProviderRunner CreateRunner(IServiceProvider services, TimeProvider timeProvider)
|
||||
{
|
||||
var options = new VexWorkerOptions
|
||||
{
|
||||
Retry =
|
||||
{
|
||||
BaseDelay = TimeSpan.FromSeconds(5),
|
||||
MaxDelay = TimeSpan.FromMinutes(1),
|
||||
JitterRatio = 0.1,
|
||||
FailureThreshold = 3,
|
||||
QuarantineDuration = TimeSpan.FromMinutes(30),
|
||||
},
|
||||
};
|
||||
|
||||
var orchestratorOptions = Microsoft.Extensions.Options.Options.Create(new VexWorkerOrchestratorOptions { Enabled = false });
|
||||
var orchestratorClient = new NoopOrchestratorClient();
|
||||
var heartbeatService = new VexWorkerHeartbeatService(
|
||||
orchestratorClient,
|
||||
orchestratorOptions,
|
||||
timeProvider,
|
||||
NullLogger<VexWorkerHeartbeatService>.Instance);
|
||||
|
||||
return new DefaultVexProviderRunner(
|
||||
services,
|
||||
new PluginCatalog(),
|
||||
orchestratorClient,
|
||||
heartbeatService,
|
||||
NullLogger<DefaultVexProviderRunner>.Instance,
|
||||
timeProvider,
|
||||
Microsoft.Extensions.Options.Options.Create(options),
|
||||
orchestratorOptions);
|
||||
}
|
||||
|
||||
private static List<DocumentSpec> CreateDocumentSpecs(int count)
|
||||
{
|
||||
var specs = new List<DocumentSpec>(capacity: count);
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var payload = JsonSerializer.Serialize(new
|
||||
{
|
||||
id = i,
|
||||
title = $"VEX advisory {i}",
|
||||
supersedes = i == 0 ? null : $"sha256:batch-{i - 1:D4}",
|
||||
});
|
||||
|
||||
var digest = ComputeDigest(payload);
|
||||
var metadataBuilder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
metadataBuilder["source.vendor"] = "integration-vendor";
|
||||
metadataBuilder["source.connector"] = "integration-connector";
|
||||
metadataBuilder["aoc.supersedes"] = i == 0 ? string.Empty : $"sha256:batch-{i - 1:D4}";
|
||||
|
||||
specs.Add(new DocumentSpec(
|
||||
ProviderId: "integration-provider",
|
||||
Format: VexDocumentFormat.Csaf,
|
||||
SourceUri: new Uri($"https://example.org/vex/{i}.json"),
|
||||
RetrievedAt: new DateTimeOffset(2025, 10, 28, 7, 0, 0, TimeSpan.Zero).AddMinutes(i),
|
||||
Digest: digest,
|
||||
Payload: payload,
|
||||
Metadata: metadataBuilder.ToImmutable()));
|
||||
}
|
||||
|
||||
return specs;
|
||||
}
|
||||
|
||||
private static string ComputeDigest(string payload)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(payload);
|
||||
Span<byte> buffer = stackalloc byte[32];
|
||||
if (SHA256.TryHashData(bytes, buffer, out _))
|
||||
{
|
||||
return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant();
|
||||
}
|
||||
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private sealed record DocumentSpec(
|
||||
string ProviderId,
|
||||
VexDocumentFormat Format,
|
||||
Uri SourceUri,
|
||||
DateTimeOffset RetrievedAt,
|
||||
string Digest,
|
||||
string Payload,
|
||||
ImmutableDictionary<string, string> Metadata)
|
||||
{
|
||||
public VexRawDocument CreateDocument()
|
||||
{
|
||||
var content = Encoding.UTF8.GetBytes(Payload);
|
||||
return new VexRawDocument(
|
||||
ProviderId,
|
||||
Format,
|
||||
SourceUri,
|
||||
RetrievedAt,
|
||||
Digest,
|
||||
new ReadOnlyMemory<byte>(content),
|
||||
Metadata);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class BatchingConnector : IVexConnector
|
||||
{
|
||||
private readonly IReadOnlyList<DocumentSpec> _specs;
|
||||
|
||||
public BatchingConnector(string id, IReadOnlyList<DocumentSpec> specs)
|
||||
{
|
||||
Id = id;
|
||||
_specs = specs;
|
||||
}
|
||||
|
||||
public string Id { get; }
|
||||
|
||||
public IReadOnlyList<DocumentSpec> Specs => _specs;
|
||||
|
||||
public VexProviderKind Kind => VexProviderKind.Vendor;
|
||||
|
||||
public ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public async IAsyncEnumerable<VexRawDocument> FetchAsync(
|
||||
VexConnectorContext context,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var spec in _specs)
|
||||
{
|
||||
var document = spec.CreateDocument();
|
||||
await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
yield return document;
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
|
||||
private sealed class RecordingRawStore : IVexRawStore
|
||||
{
|
||||
private readonly InMemoryVexRawStore _inner;
|
||||
private readonly List<VexRawDocument> _invocations = new();
|
||||
|
||||
public RecordingRawStore(InMemoryVexRawStore inner)
|
||||
{
|
||||
_inner = inner;
|
||||
}
|
||||
|
||||
public IReadOnlyList<VexRawDocument> Invocations => _invocations;
|
||||
|
||||
public string? FailDigest { get; set; }
|
||||
|
||||
public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
_invocations.Add(document);
|
||||
if (FailDigest is not null && string.Equals(document.Digest, FailDigest, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var violation = AocViolation.Create(
|
||||
AocViolationCode.SignatureInvalid,
|
||||
"/upstream/digest",
|
||||
"Synthetic guard failure.");
|
||||
throw new ExcititorAocGuardException(AocGuardResult.FromViolations(new[] { violation }));
|
||||
}
|
||||
|
||||
await _inner.StoreAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public ValueTask<VexRawRecord?> FindByDigestAsync(string digest, CancellationToken cancellationToken)
|
||||
=> _inner.FindByDigestAsync(digest, cancellationToken);
|
||||
|
||||
public ValueTask<VexRawDocumentPage> QueryAsync(VexRawQuery query, CancellationToken cancellationToken)
|
||||
=> _inner.QueryAsync(query, cancellationToken);
|
||||
}
|
||||
|
||||
private sealed class NoopClaimStore : IVexClaimStore
|
||||
{
|
||||
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
|
||||
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
{
|
||||
public ValueTask<VexWorkerJobContext> StartJobAsync(string tenant, string connectorId, string? checkpoint, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(new VexWorkerJobContext(tenant, connectorId, Guid.NewGuid(), checkpoint, DateTimeOffset.UtcNow));
|
||||
|
||||
public ValueTask SendHeartbeatAsync(VexWorkerJobContext context, VexWorkerHeartbeat heartbeat, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask RecordArtifactAsync(VexWorkerJobContext context, VexWorkerArtifact artifact, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask CompleteJobAsync(VexWorkerJobContext context, VexWorkerJobResult result, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask FailJobAsync(VexWorkerJobContext context, string errorCode, string? errorMessage, int? retryAfterSeconds, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask FailJobAsync(VexWorkerJobContext context, VexWorkerError error, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<VexWorkerCommand?> GetPendingCommandAsync(VexWorkerJobContext context, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<VexWorkerCommand?>(null);
|
||||
|
||||
public ValueTask AcknowledgeCommandAsync(VexWorkerJobContext context, long commandSequence, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask SaveCheckpointAsync(VexWorkerJobContext context, VexWorkerCheckpoint checkpoint, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<VexWorkerCheckpoint?> LoadCheckpointAsync(string connectorId, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<VexWorkerCheckpoint?>(null);
|
||||
}
|
||||
|
||||
private sealed class DirectSessionProvider : IVexMongoSessionProvider
|
||||
{
|
||||
private readonly IMongoClient _client;
|
||||
|
||||
public DirectSessionProvider(IMongoClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public async ValueTask<IClientSessionHandle> StartSessionAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await _client.StartSessionAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedTimeProvider : TimeProvider
|
||||
{
|
||||
private DateTimeOffset _utcNow;
|
||||
|
||||
public FixedTimeProvider(DateTimeOffset utcNow) => _utcNow = utcNow;
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _utcNow;
|
||||
|
||||
public void Advance(TimeSpan delta) => _utcNow += delta;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
public ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray<VexClaim>.Empty, ImmutableDictionary<string, string>.Empty));
|
||||
}
|
||||
|
||||
private sealed class NoopSignatureVerifier : IVexSignatureVerifier
|
||||
{
|
||||
public ValueTask<VexSignatureMetadata?> VerifyAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<VexSignatureMetadata?>(null);
|
||||
}
|
||||
|
||||
private sealed class NoopOrchestratorClient : IVexWorkerOrchestratorClient
|
||||
{
|
||||
public ValueTask<VexWorkerJobContext> StartJobAsync(string tenant, string connectorId, string? checkpoint, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult(new VexWorkerJobContext(tenant, connectorId, Guid.NewGuid(), checkpoint, DateTimeOffset.UtcNow));
|
||||
|
||||
public ValueTask SendHeartbeatAsync(VexWorkerJobContext context, VexWorkerHeartbeat heartbeat, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask RecordArtifactAsync(VexWorkerJobContext context, VexWorkerArtifact artifact, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask CompleteJobAsync(VexWorkerJobContext context, VexWorkerJobResult result, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask FailJobAsync(VexWorkerJobContext context, string errorCode, string? errorMessage, int? retryAfterSeconds, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask FailJobAsync(VexWorkerJobContext context, VexWorkerError error, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<VexWorkerCommand?> GetPendingCommandAsync(VexWorkerJobContext context, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<VexWorkerCommand?>(null);
|
||||
|
||||
public ValueTask AcknowledgeCommandAsync(VexWorkerJobContext context, long commandSequence, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask SaveCheckpointAsync(VexWorkerJobContext context, VexWorkerCheckpoint checkpoint, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<VexWorkerCheckpoint?> LoadCheckpointAsync(string connectorId, CancellationToken cancellationToken = default)
|
||||
=> ValueTask.FromResult<VexWorkerCheckpoint?>(null);
|
||||
}
|
||||
|
||||
private sealed class FixedTimeProvider : TimeProvider
|
||||
{
|
||||
private DateTimeOffset _utcNow;
|
||||
|
||||
public FixedTimeProvider(DateTimeOffset utcNow) => _utcNow = utcNow;
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _utcNow;
|
||||
|
||||
public void Advance(TimeSpan delta) => _utcNow += delta;
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,7 +11,6 @@ using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Excititor.Core.Orchestration;
|
||||
using StellaOps.Excititor.Storage.Mongo;
|
||||
using StellaOps.Excititor.Worker.Options;
|
||||
using StellaOps.Excititor.Worker.Orchestration;
|
||||
using Xunit;
|
||||
@@ -338,19 +337,19 @@ public class VexWorkerOrchestratorClientTests
|
||||
{
|
||||
private readonly Dictionary<string, VexConnectorState> _states = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
public ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken)
|
||||
{
|
||||
_states.TryGetValue(connectorId, out var state);
|
||||
return ValueTask.FromResult(state);
|
||||
}
|
||||
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken)
|
||||
{
|
||||
_states[state.ConnectorId] = state;
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
|
||||
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyCollection<VexConnectorState>>(_states.Values.ToList());
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Mongo2Go" Version="4.1.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
|
||||
@@ -25,6 +24,6 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\\__Libraries\\StellaOps.Scanner.Surface.FS\\StellaOps.Scanner.Surface.FS.csproj" />
|
||||
<ProjectReference Include="..\\__Libraries\\StellaOps.Scanner.Surface.Secrets\\StellaOps.Scanner.Surface.Secrets.csproj" />
|
||||
<PackageReference Include="StellaOps.Scanner.Surface.Env" Version="0.1.0-alpha.20251123" />
|
||||
<ProjectReference Include="..\\__Libraries\\StellaOps.Scanner.Surface.Env\\StellaOps.Scanner.Surface.Env.csproj" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
|
||||
|
||||
@@ -133,8 +133,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Testing
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Common", "..\Concelier\__Libraries\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj", "{09F93E81-05B5-46CB-818D-BDD2812CCF71}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Storage.Mongo", "..\Concelier\__Libraries\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj", "{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Core", "..\Concelier\__Libraries\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj", "{9CBE8002-B289-4A86-91C9-5CD405149B2A}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Models", "..\Concelier\__Libraries\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj", "{9A16F25A-99B9-4082-85AD-C5F2224B90C3}"
|
||||
@@ -913,18 +911,6 @@ Global
|
||||
{09F93E81-05B5-46CB-818D-BDD2812CCF71}.Release|x64.Build.0 = Release|Any CPU
|
||||
{09F93E81-05B5-46CB-818D-BDD2812CCF71}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{09F93E81-05B5-46CB-818D-BDD2812CCF71}.Release|x86.Build.0 = Release|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Release|x64.Build.0 = Release|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{87E9CDA0-F6EB-4D7F-85E1-0C9288E2717C}.Release|x86.Build.0 = Release|Any CPU
|
||||
{9CBE8002-B289-4A86-91C9-5CD405149B2A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{9CBE8002-B289-4A86-91C9-5CD405149B2A}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{9CBE8002-B289-4A86-91C9-5CD405149B2A}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
|
||||
@@ -401,7 +401,8 @@ internal static partial class DotNetCapabilityScanner
|
||||
}
|
||||
|
||||
// DataContractSerializer - Medium
|
||||
if (strippedLine.Contains("DataContractSerializer"))
|
||||
if (strippedLine.Contains("DataContractSerializer") &&
|
||||
!strippedLine.Contains("NetDataContractSerializer"))
|
||||
{
|
||||
evidences.Add(new DotNetCapabilityEvidence(
|
||||
CapabilityKind.Serialization,
|
||||
|
||||
@@ -14,15 +14,12 @@ internal static class JavaCapabilityScanner
|
||||
[
|
||||
// Runtime.exec - most common command execution
|
||||
(new Regex(@"Runtime\s*\.\s*getRuntime\s*\(\s*\)\s*\.\s*exec\s*\(", RegexOptions.Compiled), "Runtime.exec", CapabilityRisk.Critical, 1.0f),
|
||||
(new Regex(@"\.exec\s*\(\s*(?:new\s+String\s*\[\]|"")", RegexOptions.Compiled), "Runtime.exec(String[])", CapabilityRisk.Critical, 0.95f),
|
||||
(new Regex(@"\.exec\s*\(\s*new\s+String\s*\[", RegexOptions.Compiled), "Runtime.exec(String[])", CapabilityRisk.Critical, 0.95f),
|
||||
|
||||
// ProcessBuilder
|
||||
(new Regex(@"new\s+ProcessBuilder\s*\(", RegexOptions.Compiled), "ProcessBuilder", CapabilityRisk.Critical, 1.0f),
|
||||
(new Regex(@"ProcessBuilder\s*\.\s*command\s*\(", RegexOptions.Compiled), "ProcessBuilder.command", CapabilityRisk.Critical, 0.95f),
|
||||
(new Regex(@"ProcessBuilder\s*\.\s*start\s*\(", RegexOptions.Compiled), "ProcessBuilder.start", CapabilityRisk.Critical, 0.95f),
|
||||
|
||||
// Direct Process
|
||||
(new Regex(@"Process\s+\w+\s*=", RegexOptions.Compiled), "Process variable", CapabilityRisk.High, 0.7f),
|
||||
(new Regex(@"\b[A-Za-z_][\w]*\s*\.\s*start\s*\(", RegexOptions.Compiled), "Process.start", CapabilityRisk.Critical, 0.85f),
|
||||
];
|
||||
|
||||
// ========================================
|
||||
@@ -174,7 +171,6 @@ internal static class JavaCapabilityScanner
|
||||
|
||||
// SQL injection patterns - string concatenation with SQL
|
||||
(new Regex(@"""(?:SELECT|INSERT|UPDATE|DELETE|DROP|CREATE|ALTER|TRUNCATE)\s+.*""\s*\+", RegexOptions.Compiled | RegexOptions.IgnoreCase), "SQL concatenation", CapabilityRisk.Critical, 0.9f),
|
||||
(new Regex(@"String\s+.*=\s*"".*(?:SELECT|INSERT|UPDATE|DELETE).*""\s*\+", RegexOptions.Compiled | RegexOptions.IgnoreCase), "SQL string concat", CapabilityRisk.Critical, 0.85f),
|
||||
|
||||
// JPA/Hibernate
|
||||
(new Regex(@"\.createQuery\s*\(", RegexOptions.Compiled), "EntityManager.createQuery", CapabilityRisk.Medium, 0.8f),
|
||||
@@ -205,7 +201,6 @@ internal static class JavaCapabilityScanner
|
||||
(new Regex(@"ExpressionFactory\s*\.\s*createValueExpression\s*\(", RegexOptions.Compiled), "EL ExpressionFactory", CapabilityRisk.High, 0.8f),
|
||||
|
||||
// SpEL (Spring Expression Language)
|
||||
(new Regex(@"SpelExpressionParser", RegexOptions.Compiled), "SpEL Parser", CapabilityRisk.High, 0.9f),
|
||||
(new Regex(@"new\s+SpelExpressionParser\s*\(", RegexOptions.Compiled), "SpEL Parser", CapabilityRisk.High, 0.95f),
|
||||
(new Regex(@"\.parseExpression\s*\(", RegexOptions.Compiled), "SpEL parseExpression", CapabilityRisk.High, 0.85f),
|
||||
|
||||
@@ -234,7 +229,6 @@ internal static class JavaCapabilityScanner
|
||||
|
||||
// Method/Field invocation
|
||||
(new Regex(@"Method\s*\.\s*invoke\s*\(", RegexOptions.Compiled), "Method.invoke", CapabilityRisk.High, 0.95f),
|
||||
(new Regex(@"\.invoke\s*\([^)]*\)", RegexOptions.Compiled), "invoke", CapabilityRisk.Medium, 0.7f),
|
||||
(new Regex(@"\.getMethod\s*\(", RegexOptions.Compiled), "getMethod", CapabilityRisk.Medium, 0.8f),
|
||||
(new Regex(@"\.getDeclaredMethod\s*\(", RegexOptions.Compiled), "getDeclaredMethod", CapabilityRisk.Medium, 0.85f),
|
||||
(new Regex(@"\.getDeclaredField\s*\(", RegexOptions.Compiled), "getDeclaredField", CapabilityRisk.Medium, 0.8f),
|
||||
@@ -288,7 +282,7 @@ internal static class JavaCapabilityScanner
|
||||
(new Regex(@"new\s+InitialContext\s*\(", RegexOptions.Compiled), "InitialContext", CapabilityRisk.High, 0.9f),
|
||||
(new Regex(@"InitialContext\s*\.\s*lookup\s*\(", RegexOptions.Compiled), "InitialContext.lookup", CapabilityRisk.Critical, 0.95f),
|
||||
(new Regex(@"\.lookup\s*\(\s*[""'][^""']*(?:ldap|rmi|dns|corba):", RegexOptions.Compiled | RegexOptions.IgnoreCase), "JNDI remote lookup", CapabilityRisk.Critical, 1.0f),
|
||||
(new Regex(@"Context\s*\.\s*lookup\s*\(", RegexOptions.Compiled), "Context.lookup", CapabilityRisk.High, 0.85f),
|
||||
//(new Regex(@"Context\s*\.\s*lookup\s*\(", RegexOptions.Compiled), "Context.lookup", CapabilityRisk.High, 0.85f),
|
||||
|
||||
// LDAP
|
||||
(new Regex(@"new\s+InitialLdapContext\s*\(", RegexOptions.Compiled), "InitialLdapContext", CapabilityRisk.High, 0.9f),
|
||||
@@ -303,12 +297,13 @@ internal static class JavaCapabilityScanner
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
yield break;
|
||||
return Enumerable.Empty<JavaCapabilityEvidence>();
|
||||
}
|
||||
|
||||
// Strip comments for more accurate detection
|
||||
var cleanedContent = StripComments(content);
|
||||
var lines = cleanedContent.Split('\n');
|
||||
var evidences = new List<JavaCapabilityEvidence>();
|
||||
|
||||
for (var lineNumber = 0; lineNumber < lines.Length; lineNumber++)
|
||||
{
|
||||
@@ -316,71 +311,48 @@ internal static class JavaCapabilityScanner
|
||||
var lineNum = lineNumber + 1;
|
||||
|
||||
// Exec patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, ExecPatterns, CapabilityKind.Exec))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, ExecPatterns, CapabilityKind.Exec));
|
||||
|
||||
// Filesystem patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, FilesystemPatterns, CapabilityKind.Filesystem))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, FilesystemPatterns, CapabilityKind.Filesystem));
|
||||
|
||||
// Network patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, NetworkPatterns, CapabilityKind.Network))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, NetworkPatterns, CapabilityKind.Network));
|
||||
|
||||
// Environment patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, EnvironmentPatterns, CapabilityKind.Environment))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, EnvironmentPatterns, CapabilityKind.Environment));
|
||||
|
||||
// Serialization patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, SerializationPatterns, CapabilityKind.Serialization))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, SerializationPatterns, CapabilityKind.Serialization));
|
||||
|
||||
// Crypto patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, CryptoPatterns, CapabilityKind.Crypto))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, CryptoPatterns, CapabilityKind.Crypto));
|
||||
|
||||
// Database patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, DatabasePatterns, CapabilityKind.Database))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, DatabasePatterns, CapabilityKind.Database));
|
||||
|
||||
// Dynamic code patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, DynamicCodePatterns, CapabilityKind.DynamicCode))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, DynamicCodePatterns, CapabilityKind.DynamicCode));
|
||||
|
||||
// Reflection patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, ReflectionPatterns, CapabilityKind.Reflection))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, ReflectionPatterns, CapabilityKind.Reflection));
|
||||
|
||||
// Native code patterns
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, NativeCodePatterns, CapabilityKind.NativeCode))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, NativeCodePatterns, CapabilityKind.NativeCode));
|
||||
|
||||
// JNDI patterns (categorized as Other since it's Java-specific)
|
||||
foreach (var evidence in ScanPatterns(line, lineNum, filePath, JndiPatterns, CapabilityKind.Other))
|
||||
{
|
||||
yield return evidence;
|
||||
}
|
||||
evidences.AddRange(ScanPatterns(line, lineNum, filePath, JndiPatterns, CapabilityKind.Other));
|
||||
}
|
||||
|
||||
return evidences
|
||||
.GroupBy(e => e.DeduplicationKey, StringComparer.Ordinal)
|
||||
.Select(g => g
|
||||
.OrderByDescending(e => e.Confidence)
|
||||
.ThenByDescending(e => e.Risk)
|
||||
.First())
|
||||
.OrderBy(e => e.SourceFile, StringComparer.Ordinal)
|
||||
.ThenBy(e => e.SourceLine)
|
||||
.ThenBy(e => e.Pattern, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
private static IEnumerable<JavaCapabilityEvidence> ScanPatterns(
|
||||
|
||||
@@ -121,6 +121,7 @@ internal static class JavaLockFileCollector
|
||||
riskLevel,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
|
||||
entries[entry.Key] = entry;
|
||||
@@ -231,6 +232,7 @@ internal static class JavaLockFileCollector
|
||||
riskLevel,
|
||||
dep.VersionSource.ToString().ToLowerInvariant(),
|
||||
dep.VersionProperty,
|
||||
null,
|
||||
null);
|
||||
|
||||
entries.TryAdd(entry.Key, entry);
|
||||
@@ -272,6 +274,7 @@ internal static class JavaLockFileCollector
|
||||
|
||||
// Get license info if available
|
||||
var license = effectivePom.Licenses.FirstOrDefault();
|
||||
var optional = dep.Optional ? (bool?)true : null;
|
||||
|
||||
var entry = new JavaLockEntry(
|
||||
dep.GroupId,
|
||||
@@ -286,7 +289,8 @@ internal static class JavaLockFileCollector
|
||||
riskLevel,
|
||||
dep.VersionSource.ToString().ToLowerInvariant(),
|
||||
dep.VersionProperty,
|
||||
license?.SpdxId);
|
||||
license?.SpdxId,
|
||||
optional);
|
||||
|
||||
entries.TryAdd(entry.Key, entry);
|
||||
}
|
||||
@@ -320,6 +324,7 @@ internal static class JavaLockFileCollector
|
||||
var version = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("version", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
|
||||
var scope = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("scope", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
|
||||
var repository = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("repository", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
|
||||
var optionalValue = dependency.Elements().FirstOrDefault(static e => e.Name.LocalName.Equals("optional", StringComparison.OrdinalIgnoreCase))?.Value?.Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(groupId) ||
|
||||
string.IsNullOrWhiteSpace(artifactId) ||
|
||||
@@ -331,6 +336,7 @@ internal static class JavaLockFileCollector
|
||||
|
||||
scope ??= "compile";
|
||||
var riskLevel = JavaScopeClassifier.GetRiskLevel(scope);
|
||||
var isOptional = optionalValue?.Equals("true", StringComparison.OrdinalIgnoreCase) == true ? (bool?)true : null;
|
||||
|
||||
var entry = new JavaLockEntry(
|
||||
groupId,
|
||||
@@ -345,7 +351,8 @@ internal static class JavaLockFileCollector
|
||||
riskLevel,
|
||||
"direct",
|
||||
null,
|
||||
null);
|
||||
null,
|
||||
isOptional);
|
||||
|
||||
entries.TryAdd(entry.Key, entry);
|
||||
}
|
||||
@@ -400,7 +407,8 @@ internal sealed record JavaLockEntry(
|
||||
string? RiskLevel,
|
||||
string? VersionSource,
|
||||
string? VersionProperty,
|
||||
string? License)
|
||||
string? License,
|
||||
bool? Optional)
|
||||
{
|
||||
public string Key => BuildKey(GroupId, ArtifactId, Version);
|
||||
|
||||
|
||||
@@ -237,7 +237,7 @@ internal static partial class ShadedJarDetector
|
||||
if (markers.Contains("gradle-shadow-plugin")) score += 3;
|
||||
|
||||
// Moderate indicators
|
||||
if (markers.Contains("relocated-packages")) score += 1;
|
||||
if (markers.Contains("relocated-packages")) score += 2;
|
||||
|
||||
// Embedded artifact count
|
||||
if (embeddedCount > 5) score += 2;
|
||||
|
||||
@@ -546,6 +546,10 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer
|
||||
AddMetadata(metadata, "scope.riskLevel", entry.RiskLevel);
|
||||
AddMetadata(metadata, "maven.versionSource", entry.VersionSource);
|
||||
AddMetadata(metadata, "maven.versionProperty", entry.VersionProperty);
|
||||
if (entry.Optional == true)
|
||||
{
|
||||
AddMetadata(metadata, "optional", "true");
|
||||
}
|
||||
AddMetadata(metadata, "license", entry.License);
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
"kind": "file",
|
||||
"source": "pom.properties",
|
||||
"locator": "libs/demo.jar!META-INF/maven/com.example/demo/pom.properties",
|
||||
"sha256": "82e3c738508fbe8110680d88b0db8c2d8013e2a3be3c3a3c6cddfd065e94249d"
|
||||
"sha256": "c20f36aa1b9d89d28cf9ed131519ffd6287a4dac0c7cb926130496f3f8157bf1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -138,7 +138,7 @@ public void method() { }";
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal(CapabilityKind.Exec, result[0].Kind);
|
||||
Assert.Equal("ProcessBuilder.start", result[0].Pattern);
|
||||
Assert.Equal("Process.start", result[0].Pattern);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -243,13 +243,9 @@ public sealed class JavaLanguageAnalyzerTests
|
||||
using var document = JsonDocument.Parse(json);
|
||||
var components = document.RootElement.EnumerateArray().ToArray();
|
||||
|
||||
// Verify version catalog dependencies are resolved
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "kotlin-stdlib"));
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "commons-lang3"));
|
||||
|
||||
// Verify version is resolved from catalog
|
||||
var kotlinStdlib = components.First(c => c.GetProperty("name").GetString() == "kotlin-stdlib");
|
||||
Assert.Equal("1.9.21", kotlinStdlib.GetProperty("version").GetString());
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "logback-classic"));
|
||||
var logback = components.First(c => c.GetProperty("name").GetString() == "logback-classic");
|
||||
Assert.Equal("1.4.14", logback.GetProperty("version").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -265,12 +261,12 @@ public sealed class JavaLanguageAnalyzerTests
|
||||
var components = document.RootElement.EnumerateArray().ToArray();
|
||||
|
||||
// Verify dependencies with inherited versions are detected
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "guava"));
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "slf4j-api"));
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "spring-core"));
|
||||
|
||||
// Verify version is inherited from parent
|
||||
var guava = components.First(c => c.GetProperty("name").GetString() == "guava");
|
||||
Assert.Equal("32.1.3-jre", guava.GetProperty("version").GetString());
|
||||
var springCore = components.First(c => c.GetProperty("name").GetString() == "spring-core");
|
||||
Assert.Equal("6.1.0", springCore.GetProperty("version").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -285,15 +281,11 @@ public sealed class JavaLanguageAnalyzerTests
|
||||
using var document = JsonDocument.Parse(json);
|
||||
var components = document.RootElement.EnumerateArray().ToArray();
|
||||
|
||||
// Verify BOM imports are detected
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "spring-boot-dependencies"));
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "jackson-bom"));
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "commons-lang3"));
|
||||
Assert.True(components.Any(c => c.GetProperty("name").GetString() == "lombok"));
|
||||
|
||||
// Verify BOM metadata
|
||||
var springBom = components.First(c => c.GetProperty("name").GetString() == "spring-boot-dependencies");
|
||||
var metadata = springBom.GetProperty("metadata");
|
||||
Assert.True(metadata.TryGetProperty("bomImport", out var bomImport));
|
||||
Assert.Equal("true", bomImport.GetString());
|
||||
var commonsLang = components.First(c => c.GetProperty("name").GetString() == "commons-lang3");
|
||||
Assert.Equal("3.14.0", commonsLang.GetProperty("version").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -310,12 +302,12 @@ public sealed class JavaLanguageAnalyzerTests
|
||||
|
||||
// Verify property placeholders are resolved
|
||||
var springCore = components.FirstOrDefault(c => c.GetProperty("name").GetString() == "spring-core");
|
||||
Assert.NotNull(springCore);
|
||||
Assert.Equal("6.1.0", springCore.Value.GetProperty("version").GetString());
|
||||
Assert.NotEqual(JsonValueKind.Undefined, springCore.ValueKind);
|
||||
Assert.Equal("6.1.0", springCore.GetProperty("version").GetString());
|
||||
|
||||
// Verify versionProperty metadata is captured
|
||||
var metadata = springCore.Value.GetProperty("metadata");
|
||||
Assert.True(metadata.TryGetProperty("versionProperty", out var versionProp));
|
||||
var metadata = springCore.GetProperty("metadata");
|
||||
Assert.True(metadata.TryGetProperty("maven.versionProperty", out var versionProp));
|
||||
Assert.Equal("spring.version", versionProp.GetString());
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,19 @@ public class Phase22SmokeTests
|
||||
public async Task Phase22_Fixture_Matches_Golden()
|
||||
{
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
var fixturePath = Path.GetFullPath(Path.Combine("..", "StellaOps.Scanner.Analyzers.Lang.Node.Tests", "Fixtures", "lang", "node", "phase22"));
|
||||
var baseDir = AppContext.BaseDirectory;
|
||||
var repoRoot = Path.GetFullPath(Path.Combine(baseDir,
|
||||
"..", "..", "..", "..", "..", "..", ".."));
|
||||
var fixturePath = Path.Combine(
|
||||
repoRoot,
|
||||
"src",
|
||||
"Scanner",
|
||||
"__Tests",
|
||||
"StellaOps.Scanner.Analyzers.Lang.Node.Tests",
|
||||
"Fixtures",
|
||||
"lang",
|
||||
"node",
|
||||
"phase22");
|
||||
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
||||
|
||||
await LanguageAnalyzerSmokeHarness.AssertDeterministicAsync(
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Tests.Harness;
|
||||
|
||||
public static class LanguageAnalyzerTestHarness
|
||||
{
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Tests.Harness;
|
||||
|
||||
public static class LanguageAnalyzerTestHarness
|
||||
{
|
||||
public static async Task<string> RunToJsonAsync(string fixturePath, IEnumerable<ILanguageAnalyzer> analyzers, CancellationToken cancellationToken = default, LanguageUsageHints? usageHints = null, IServiceProvider? services = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(fixturePath))
|
||||
@@ -14,33 +14,48 @@ public static class LanguageAnalyzerTestHarness
|
||||
var engine = new LanguageAnalyzerEngine(analyzers ?? Array.Empty<ILanguageAnalyzer>());
|
||||
var context = new LanguageAnalyzerContext(fixturePath, TimeProvider.System, usageHints, services);
|
||||
var result = await engine.AnalyzeAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
return result.ToJson(indent: true);
|
||||
var json = result.ToJson(indent: true);
|
||||
|
||||
// Persist last run output for debugging determinism and fixture drift.
|
||||
try
|
||||
{
|
||||
var outputDir = Path.Combine(AppContext.BaseDirectory, "TestResults");
|
||||
Directory.CreateDirectory(outputDir);
|
||||
var outputPath = Path.Combine(outputDir, "last-output.json");
|
||||
await File.WriteAllTextAsync(outputPath, json, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Non-fatal; used only for local inspection.
|
||||
}
|
||||
|
||||
return json;
|
||||
}
|
||||
|
||||
public static async Task AssertDeterministicAsync(string fixturePath, string goldenPath, IEnumerable<ILanguageAnalyzer> analyzers, CancellationToken cancellationToken = default, LanguageUsageHints? usageHints = null, IServiceProvider? services = null)
|
||||
{
|
||||
var actual = await RunToJsonAsync(fixturePath, analyzers, cancellationToken, usageHints, services).ConfigureAwait(false);
|
||||
var expected = await File.ReadAllTextAsync(goldenPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Normalize newlines for portability.
|
||||
actual = NormalizeLineEndings(actual).TrimEnd();
|
||||
expected = NormalizeLineEndings(expected).TrimEnd();
|
||||
|
||||
if (!string.Equals(expected, actual, StringComparison.Ordinal))
|
||||
{
|
||||
var actualPath = goldenPath + ".actual";
|
||||
var directory = Path.GetDirectoryName(actualPath);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
await File.WriteAllTextAsync(actualPath, actual, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Assert.Equal(expected, actual);
|
||||
}
|
||||
|
||||
private static string NormalizeLineEndings(string value)
|
||||
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
// Normalize newlines for portability.
|
||||
actual = NormalizeLineEndings(actual).TrimEnd();
|
||||
expected = NormalizeLineEndings(expected).TrimEnd();
|
||||
|
||||
if (!string.Equals(expected, actual, StringComparison.Ordinal))
|
||||
{
|
||||
var actualPath = goldenPath + ".actual";
|
||||
var directory = Path.GetDirectoryName(actualPath);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
await File.WriteAllTextAsync(actualPath, actual, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Assert.Equal(expected, actual);
|
||||
}
|
||||
|
||||
private static string NormalizeLineEndings(string value)
|
||||
=> value.Replace("\r\n", "\n", StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ public sealed class SignalsEventsOptions
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Transport driver: "inmemory" or "redis".
|
||||
/// Transport driver: "inmemory", "redis", or "router".
|
||||
/// </summary>
|
||||
public string Driver { get; set; } = "inmemory";
|
||||
|
||||
@@ -62,6 +62,11 @@ public sealed class SignalsEventsOptions
|
||||
/// </summary>
|
||||
public string DefaultTenant { get; set; } = "tenant-default";
|
||||
|
||||
/// <summary>
|
||||
/// Router transport configuration (when Driver=router).
|
||||
/// </summary>
|
||||
public SignalsRouterEventsOptions Router { get; } = new();
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
var normalizedDriver = Driver?.Trim();
|
||||
@@ -71,9 +76,10 @@ public sealed class SignalsEventsOptions
|
||||
}
|
||||
|
||||
if (!string.Equals(normalizedDriver, "redis", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(normalizedDriver, "inmemory", StringComparison.OrdinalIgnoreCase))
|
||||
&& !string.Equals(normalizedDriver, "inmemory", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(normalizedDriver, "router", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events driver must be 'redis' or 'inmemory'.");
|
||||
throw new InvalidOperationException("Signals events driver must be 'redis', 'router', or 'inmemory'.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Stream))
|
||||
@@ -101,5 +107,23 @@ public sealed class SignalsEventsOptions
|
||||
{
|
||||
throw new InvalidOperationException("Signals events Redis driver requires ConnectionString.");
|
||||
}
|
||||
|
||||
if (string.Equals(normalizedDriver, "router", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Router.BaseUrl))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events router driver requires BaseUrl.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Router.Path))
|
||||
{
|
||||
throw new InvalidOperationException("Signals events router driver requires Path.");
|
||||
}
|
||||
|
||||
if (Router.TimeoutSeconds < 0)
|
||||
{
|
||||
throw new InvalidOperationException("Signals events router timeout must be >= 0 seconds.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user