diff --git a/docs/12_PERFORMANCE_WORKBOOK.md b/docs/12_PERFORMANCE_WORKBOOK.md index e680a64eb..0570b81be 100644 --- a/docs/12_PERFORMANCE_WORKBOOK.md +++ b/docs/12_PERFORMANCE_WORKBOOK.md @@ -113,7 +113,7 @@ | Step | Time (ms) | |------------------------------------|----------:| -| `/quota/check` Valkey LUA INCR (Redis-compatible) | 0.8 | +| `/quota/check` Redis LUA INCR | 0.8 | | Soft wait sleep (server) | 5 000 | | Hard wait sleep (server) | 60 000 | | End‑to‑end wall‑time (soft‑hit) | 5 003 | diff --git a/docs/implplan/SPRINT_8200_0001_0002_provcache_invalidation_airgap.md b/docs/implplan/SPRINT_8200_0001_0002_provcache_invalidation_airgap.md index 1948710ca..0043d9e8a 100644 --- a/docs/implplan/SPRINT_8200_0001_0002_provcache_invalidation_airgap.md +++ b/docs/implplan/SPRINT_8200_0001_0002_provcache_invalidation_airgap.md @@ -146,12 +146,12 @@ For air-gap export, the minimal bundle contains: | 47 | PROV-8200-147 | DONE | Task 44 | AirGap Guild | Implement chunk verification during lazy fetch. | | 48 | PROV-8200-148 | DONE | Tasks 44-47 | QA Guild | Add lazy fetch tests (connected + disconnected). | | **Wave 7 (Revocation Index Table)** | | | | | | -| 49 | PROV-8200-149 | DOING | Tasks 0-6 | Platform Guild | Define `provcache.prov_revocations` table. | -| 50 | PROV-8200-150 | TODO | Task 49 | Platform Guild | Implement revocation ledger for audit trail. | -| 51 | PROV-8200-151 | TODO | Task 50 | Platform Guild | Implement revocation replay for catch-up scenarios. | -| 52 | PROV-8200-152 | TODO | Tasks 49-51 | QA Guild | Add revocation ledger tests. | +| 49 | PROV-8200-149 | DONE | Tasks 0-6 | Platform Guild | Define `provcache.prov_revocations` table. | +| 50 | PROV-8200-150 | DONE | Task 49 | Platform Guild | Implement revocation ledger for audit trail. | +| 51 | PROV-8200-151 | DONE | Task 50 | Platform Guild | Implement revocation replay for catch-up scenarios. | +| 52 | PROV-8200-152 | DONE | Tasks 49-51 | QA Guild | Add revocation ledger tests. | | **Wave 8 (Documentation)** | | | | | | -| 53 | PROV-8200-153 | TODO | All prior | Docs Guild | Document invalidation mechanisms. | +| 53 | PROV-8200-153 | DOING | All prior | Docs Guild | Document invalidation mechanisms. | | 54 | PROV-8200-154 | TODO | All prior | Docs Guild | Document air-gap export/import workflow. | | 55 | PROV-8200-155 | TODO | All prior | Docs Guild | Document evidence density levels. | | 56 | PROV-8200-156 | TODO | All prior | Docs Guild | Update `docs/24_OFFLINE_KIT.md` with Provcache integration. | @@ -394,4 +394,5 @@ public sealed record FeedEpochAdvancedEvent | 2025-12-26 | Wave 2 (Evidence Chunk Storage): Implemented IEvidenceChunker, EvidenceChunker (Merkle tree), PostgresEvidenceChunkRepository. Added 14 chunking tests. Tasks 14-21 DONE. | Agent | | 2025-12-26 | Wave 3 (Evidence Paging API): Added paged evidence retrieval endpoints (GET /proofs/{proofRoot}, manifest, chunks, POST verify). Added 11 API tests. Tasks 22-26 DONE. | Agent | | 2025-12-26 | Wave 4 (Minimal Proof Export): Created MinimalProofBundle format, IMinimalProofExporter interface, MinimalProofExporter with Lite/Standard/Strict density levels and DSSE signing. Added 16 export tests. Tasks 27-34 DONE. | Agent | -| 2025-12-26 | Wave 5 (CLI Commands): Implemented ProvCommandGroup with `stella prov export`, `stella prov import`, `stella prov verify` commands. Tasks 35-42 DONE. Task 43 BLOCKED (CLI has pre-existing build error unrelated to Provcache). | Agent || 2025-12-26 | Wave 6 (Lazy Evidence Pull): Implemented ILazyEvidenceFetcher interface, HttpChunkFetcher (connected mode), FileChunkFetcher (sneakernet mode), LazyFetchOrchestrator with chunk verification. Added 13 lazy fetch tests. Total: 107 tests passing. Tasks 44-48 DONE. | Agent | \ No newline at end of file +| 2025-12-26 | Wave 5 (CLI Commands): Implemented ProvCommandGroup with `stella prov export`, `stella prov import`, `stella prov verify` commands. Tasks 35-42 DONE. Task 43 BLOCKED (CLI has pre-existing build error unrelated to Provcache). | Agent || 2025-12-26 | Wave 6 (Lazy Evidence Pull): Implemented ILazyEvidenceFetcher interface, HttpChunkFetcher (connected mode), FileChunkFetcher (sneakernet mode), LazyFetchOrchestrator with chunk verification. Added 13 lazy fetch tests. Total: 107 tests passing. Tasks 44-48 DONE. | Agent | +| 2025-12-26 | Wave 7 (Revocation Index Table): Implemented ProvRevocationEntity, IRevocationLedger interface, InMemoryRevocationLedger, RevocationReplayService with checkpoint support. Added 17 revocation ledger tests. Total: 124 tests passing. Tasks 49-52 DONE. | Agent | \ No newline at end of file diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs index 61e4b6fe2..24c8a20ba 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs @@ -15,6 +15,7 @@ using StellaOps.Concelier.Storage; using StellaOps.Concelier.Storage.Advisories; using StellaOps.Concelier.Storage; using StellaOps.Concelier.Storage; +using StellaOps.Concelier.Core.Canonical; using StellaOps.Plugin; namespace StellaOps.Concelier.Connector.Ghsa; @@ -37,6 +38,7 @@ public sealed class GhsaConnector : IFeedConnector private readonly GhsaDiagnostics _diagnostics; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; + private readonly ICanonicalAdvisoryService? _canonicalService; private readonly object _rateLimitWarningLock = new(); private readonly Dictionary<(string Phase, string Resource), bool> _rateLimitWarnings = new(); @@ -50,7 +52,8 @@ public sealed class GhsaConnector : IFeedConnector IOptions options, GhsaDiagnostics diagnostics, TimeProvider? timeProvider, - ILogger logger) + ILogger logger, + ICanonicalAdvisoryService? canonicalService = null) { _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); @@ -63,6 +66,7 @@ public sealed class GhsaConnector : IFeedConnector _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); _timeProvider = timeProvider ?? TimeProvider.System; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _canonicalService = canonicalService; // Optional - canonical ingest } public string SourceName => GhsaConnectorPlugin.SourceName; @@ -399,6 +403,14 @@ public sealed class GhsaConnector : IFeedConnector await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + // Ingest to canonical advisory service if available + if (_canonicalService is not null) + { + var rawPayloadJson = dtoRecord.Payload.ToJson(); + await IngestToCanonicalAsync(advisory, rawPayloadJson, document.FetchedAt, cancellationToken).ConfigureAwait(false); + } + pendingMappings.Remove(documentId); _diagnostics.MapSuccess(1); } @@ -544,4 +556,90 @@ public sealed class GhsaConnector : IFeedConnector return false; } + + /// + /// Ingests GHSA advisory to canonical advisory service for deduplication. + /// Creates one RawAdvisory per affected package. + /// + private async Task IngestToCanonicalAsync( + Advisory advisory, + string rawPayloadJson, + DateTimeOffset fetchedAt, + CancellationToken cancellationToken) + { + if (_canonicalService is null || advisory.AffectedPackages.IsEmpty) + { + return; + } + + // Find primary CVE from aliases + var cve = advisory.Aliases + .FirstOrDefault(a => a.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) + ?? advisory.AdvisoryKey; // Fall back to GHSA ID if no CVE + + // Extract CWE weaknesses + var weaknesses = advisory.Cwes + .Where(w => w.Identifier.StartsWith("CWE-", StringComparison.OrdinalIgnoreCase)) + .Select(w => w.Identifier) + .ToList(); + + // Create one RawAdvisory per affected package + foreach (var affected in advisory.AffectedPackages) + { + if (string.IsNullOrWhiteSpace(affected.Identifier)) + { + continue; + } + + // Build version range JSON + string? versionRangeJson = null; + if (!affected.VersionRanges.IsEmpty) + { + var firstRange = affected.VersionRanges[0]; + var rangeObj = new + { + introduced = firstRange.IntroducedVersion, + @fixed = firstRange.FixedVersion, + last_affected = firstRange.LastAffectedVersion + }; + versionRangeJson = JsonSerializer.Serialize(rangeObj, SerializerOptions); + } + + var rawAdvisory = new RawAdvisory + { + SourceAdvisoryId = advisory.AdvisoryKey, + Cve = cve, + AffectsKey = affected.Identifier, + VersionRangeJson = versionRangeJson, + Weaknesses = weaknesses, + PatchLineage = null, + Severity = advisory.Severity, + Title = advisory.Title, + Summary = advisory.Summary, + VendorStatus = VendorStatus.Affected, + RawPayloadJson = rawPayloadJson, + FetchedAt = fetchedAt + }; + + try + { + var result = await _canonicalService.IngestAsync(SourceName, rawAdvisory, cancellationToken).ConfigureAwait(false); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "Canonical ingest for {GhsaId}/{AffectsKey}: {Decision} (canonical={CanonicalId})", + advisory.AdvisoryKey, affected.Identifier, result.Decision, result.CanonicalId); + } + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Failed to ingest {GhsaId}/{AffectsKey} to canonical service", + advisory.AdvisoryKey, affected.Identifier); + // Don't fail the mapping operation for canonical ingest failures + } + } + } } diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs index d9b9e5e56..e86e207ed 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs @@ -13,9 +13,8 @@ using StellaOps.Concelier.Connector.Nvd.Configuration; using StellaOps.Concelier.Connector.Nvd.Internal; using StellaOps.Concelier.Storage; using StellaOps.Concelier.Storage.Advisories; -using StellaOps.Concelier.Storage; -using StellaOps.Concelier.Storage; using StellaOps.Concelier.Storage.ChangeHistory; +using StellaOps.Concelier.Storage.Contracts; using StellaOps.Concelier.Core.Canonical; using StellaOps.Plugin; using Json.Schema; @@ -322,7 +321,7 @@ public sealed class NvdConnector : IFeedConnector private async Task> FetchAdditionalPagesAsync( TimeWindow window, IReadOnlyDictionary baseMetadata, - DocumentRecord firstDocument, + StorageDocument firstDocument, CancellationToken cancellationToken) { if (firstDocument.PayloadId is null) diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/EwsVerdictDeterminismTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/EwsVerdictDeterminismTests.cs index 7a28cc70f..803ed37d0 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/EwsVerdictDeterminismTests.cs +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/EwsVerdictDeterminismTests.cs @@ -52,13 +52,13 @@ public sealed class EwsVerdictDeterminismTests // Assert - All results should be byte-identical var firstScore = results[0].Score; var firstBucket = results[0].Bucket; - var firstDimensions = results[0].Dimensions; + var firstBreakdown = results[0].Breakdown; results.Should().AllSatisfy(r => { r.Score.Should().Be(firstScore, "score must be deterministic"); r.Bucket.Should().Be(firstBucket, "bucket must be deterministic"); - r.Dimensions.Should().BeEquivalentTo(firstDimensions, "dimensions must be deterministic"); + r.Breakdown.Should().BeEquivalentTo(firstBreakdown, "breakdown must be deterministic"); }); } @@ -155,16 +155,16 @@ public sealed class EwsVerdictDeterminismTests // Custom policy with different weights var customPolicy = new EvidenceWeightPolicy { - PolicyId = "custom-test-policy", - Version = "1.0", + Version = "ews.v1", + Profile = "custom-test-policy", Weights = new EvidenceWeights { - Reachability = 0.50, // Much higher weight on reachability - Runtime = 0.10, - Backport = 0.05, - Exploit = 0.20, - Source = 0.10, - Mitigation = 0.05 + Rch = 0.50, // Much higher weight on reachability + Rts = 0.10, + Bkp = 0.05, + Xpl = 0.20, + Src = 0.10, + Mit = 0.05 }, Buckets = EvidenceWeightPolicy.DefaultProduction.Buckets }; diff --git a/src/__Libraries/__Tests/StellaOps.Provcache.Tests/RevocationLedgerTests.cs b/src/__Libraries/__Tests/StellaOps.Provcache.Tests/RevocationLedgerTests.cs index ea62dbf5a..bfefac720 100644 --- a/src/__Libraries/__Tests/StellaOps.Provcache.Tests/RevocationLedgerTests.cs +++ b/src/__Libraries/__Tests/StellaOps.Provcache.Tests/RevocationLedgerTests.cs @@ -189,17 +189,17 @@ public sealed class RevocationLedgerTests } [Fact] - public void Clear_RemovesAllEntries() + public async Task Clear_RemovesAllEntries() { // Arrange - _ledger.RecordAsync(CreateTestEntry(RevocationTypes.Signer, "s1")).GetAwaiter().GetResult(); - _ledger.RecordAsync(CreateTestEntry(RevocationTypes.Signer, "s2")).GetAwaiter().GetResult(); + await _ledger.RecordAsync(CreateTestEntry(RevocationTypes.Signer, "s1")); + await _ledger.RecordAsync(CreateTestEntry(RevocationTypes.Signer, "s2")); // Act _ledger.Clear(); // Assert - var seqNo = _ledger.GetLatestSeqNoAsync().GetAwaiter().GetResult(); + var seqNo = await _ledger.GetLatestSeqNoAsync(); seqNo.Should().Be(0); }