stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

Merge branch 'main' of https://git.stella-ops.org/stella-ops.org/git.stella-ops.org

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-25 19:58:42 +02:00
parent b8b2d83f4a 82a49f6743
commit 4dfa1b8e05
6 changed files with 123 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/12_PERFORMANCE_WORKBOOK.md
View File

@@ -113,7 +113,7 @@
| Step | Time (ms) |
|------------------------------------|----------:|
| `/quota/check` Valkey LUA INCR (Redis-compatible) | 0.8 |
| `/quota/check` Redis LUA INCR | 0.8 |
| Soft wait sleep (server) | 5 000 |
| Hard wait sleep (server) | 60 000 |
| End‑to‑end wall‑time (soft‑hit) | 5 003 |

11
docs/implplan/SPRINT_8200_0001_0002_provcache_invalidation_airgap.md
View File

@@ -146,12 +146,12 @@ For air-gap export, the minimal bundle contains:
| 47 | PROV-8200-147 | DONE | Task 44 | AirGap Guild | Implement chunk verification during lazy fetch. |
| 48 | PROV-8200-148 | DONE | Tasks 44-47 | QA Guild | Add lazy fetch tests (connected + disconnected). |
| **Wave 7 (Revocation Index Table)** | | | | | |
| 49 | PROV-8200-149 | DOING | Tasks 0-6 | Platform Guild | Define `provcache.prov_revocations` table. |
| 50 | PROV-8200-150 | TODO | Task 49 | Platform Guild | Implement revocation ledger for audit trail. |
| 51 | PROV-8200-151 | TODO | Task 50 | Platform Guild | Implement revocation replay for catch-up scenarios. |
| 52 | PROV-8200-152 | TODO | Tasks 49-51 | QA Guild | Add revocation ledger tests. |
| 49 | PROV-8200-149 | DONE | Tasks 0-6 | Platform Guild | Define `provcache.prov_revocations` table. |
| 50 | PROV-8200-150 | DONE | Task 49 | Platform Guild | Implement revocation ledger for audit trail. |
| 51 | PROV-8200-151 | DONE | Task 50 | Platform Guild | Implement revocation replay for catch-up scenarios. |
| 52 | PROV-8200-152 | DONE | Tasks 49-51 | QA Guild | Add revocation ledger tests. |
| **Wave 8 (Documentation)** | | | | | |
| 53 | PROV-8200-153 | TODO | All prior | Docs Guild | Document invalidation mechanisms. |
| 53 | PROV-8200-153 | DOING | All prior | Docs Guild | Document invalidation mechanisms. |
| 54 | PROV-8200-154 | TODO | All prior | Docs Guild | Document air-gap export/import workflow. |
| 55 | PROV-8200-155 | TODO | All prior | Docs Guild | Document evidence density levels. |
| 56 | PROV-8200-156 | TODO | All prior | Docs Guild | Update `docs/24_OFFLINE_KIT.md` with Provcache integration. |
@@ -395,3 +395,4 @@ public sealed record FeedEpochAdvancedEvent
| 2025-12-26 | Wave 3 (Evidence Paging API): Added paged evidence retrieval endpoints (GET /proofs/{proofRoot}, manifest, chunks, POST verify). Added 11 API tests. Tasks 22-26 DONE. | Agent |
| 2025-12-26 | Wave 4 (Minimal Proof Export): Created MinimalProofBundle format, IMinimalProofExporter interface, MinimalProofExporter with Lite/Standard/Strict density levels and DSSE signing. Added 16 export tests. Tasks 27-34 DONE. | Agent |
| 2025-12-26 | Wave 5 (CLI Commands): Implemented ProvCommandGroup with `stella prov export`, `stella prov import`, `stella prov verify` commands. Tasks 35-42 DONE. Task 43 BLOCKED (CLI has pre-existing build error unrelated to Provcache). | Agent || 2025-12-26 | Wave 6 (Lazy Evidence Pull): Implemented ILazyEvidenceFetcher interface, HttpChunkFetcher (connected mode), FileChunkFetcher (sneakernet mode), LazyFetchOrchestrator with chunk verification. Added 13 lazy fetch tests. Total: 107 tests passing. Tasks 44-48 DONE. | Agent |
| 2025-12-26 | Wave 7 (Revocation Index Table): Implemented ProvRevocationEntity, IRevocationLedger interface, InMemoryRevocationLedger, RevocationReplayService with checkpoint support. Added 17 revocation ledger tests. Total: 124 tests passing. Tasks 49-52 DONE. | Agent |

100
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs
View File

@@ -15,6 +15,7 @@ using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Ghsa;
@@ -37,6 +38,7 @@ public sealed class GhsaConnector : IFeedConnector
private readonly GhsaDiagnostics _diagnostics;
private readonly TimeProvider _timeProvider;
private readonly ILogger<GhsaConnector> _logger;
private readonly ICanonicalAdvisoryService? _canonicalService;
private readonly object _rateLimitWarningLock = new();
private readonly Dictionary<(string Phase, string Resource), bool> _rateLimitWarnings = new();
@@ -50,7 +52,8 @@ public sealed class GhsaConnector : IFeedConnector
IOptions<GhsaOptions> options,
GhsaDiagnostics diagnostics,
TimeProvider? timeProvider,
ILogger<GhsaConnector> logger)
ILogger<GhsaConnector> logger,
ICanonicalAdvisoryService? canonicalService = null)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
@@ -63,6 +66,7 @@ public sealed class GhsaConnector : IFeedConnector
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_canonicalService = canonicalService; // Optional - canonical ingest
}
public string SourceName => GhsaConnectorPlugin.SourceName;
@@ -399,6 +403,14 @@ public sealed class GhsaConnector : IFeedConnector
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
// Ingest to canonical advisory service if available
if (_canonicalService is not null)
{
var rawPayloadJson = dtoRecord.Payload.ToJson();
await IngestToCanonicalAsync(advisory, rawPayloadJson, document.FetchedAt, cancellationToken).ConfigureAwait(false);
}
pendingMappings.Remove(documentId);
_diagnostics.MapSuccess(1);
}
@@ -544,4 +556,90 @@ public sealed class GhsaConnector : IFeedConnector
return false;
}
/// <summary>
/// Ingests GHSA advisory to canonical advisory service for deduplication.
/// Creates one RawAdvisory per affected package.
/// </summary>
private async Task IngestToCanonicalAsync(
Advisory advisory,
string rawPayloadJson,
DateTimeOffset fetchedAt,
CancellationToken cancellationToken)
{
if (_canonicalService is null || advisory.AffectedPackages.IsEmpty)
{
return;
}
// Find primary CVE from aliases
var cve = advisory.Aliases
.FirstOrDefault(a => a.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
?? advisory.AdvisoryKey; // Fall back to GHSA ID if no CVE
// Extract CWE weaknesses
var weaknesses = advisory.Cwes
.Where(w => w.Identifier.StartsWith("CWE-", StringComparison.OrdinalIgnoreCase))
.Select(w => w.Identifier)
.ToList();
// Create one RawAdvisory per affected package
foreach (var affected in advisory.AffectedPackages)
{
if (string.IsNullOrWhiteSpace(affected.Identifier))
{
continue;
}
// Build version range JSON
string? versionRangeJson = null;
if (!affected.VersionRanges.IsEmpty)
{
var firstRange = affected.VersionRanges[0];
var rangeObj = new
{
introduced = firstRange.IntroducedVersion,
@fixed = firstRange.FixedVersion,
last_affected = firstRange.LastAffectedVersion
};
versionRangeJson = JsonSerializer.Serialize(rangeObj, SerializerOptions);
}
var rawAdvisory = new RawAdvisory
{
SourceAdvisoryId = advisory.AdvisoryKey,
Cve = cve,
AffectsKey = affected.Identifier,
VersionRangeJson = versionRangeJson,
Weaknesses = weaknesses,
PatchLineage = null,
Severity = advisory.Severity,
Title = advisory.Title,
Summary = advisory.Summary,
VendorStatus = VendorStatus.Affected,
RawPayloadJson = rawPayloadJson,
FetchedAt = fetchedAt
};
try
{
var result = await _canonicalService.IngestAsync(SourceName, rawAdvisory, cancellationToken).ConfigureAwait(false);
if (_logger.IsEnabled(LogLevel.Debug))
{
_logger.LogDebug(
"Canonical ingest for {GhsaId}/{AffectsKey}: {Decision} (canonical={CanonicalId})",
advisory.AdvisoryKey, affected.Identifier, result.Decision, result.CanonicalId);
}
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Failed to ingest {GhsaId}/{AffectsKey} to canonical service",
advisory.AdvisoryKey, affected.Identifier);
// Don't fail the mapping operation for canonical ingest failures
}
}
}
}

5
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs
View File

@@ -13,9 +13,8 @@ using StellaOps.Concelier.Connector.Nvd.Configuration;
using StellaOps.Concelier.Connector.Nvd.Internal;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.Advisories;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage;
using StellaOps.Concelier.Storage.ChangeHistory;
using StellaOps.Concelier.Storage.Contracts;
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Plugin;
using Json.Schema;
@@ -322,7 +321,7 @@ public sealed class NvdConnector : IFeedConnector
private async Task<IReadOnlyCollection<Guid>> FetchAdditionalPagesAsync(
TimeWindow window,
IReadOnlyDictionary<string, string> baseMetadata,
DocumentRecord firstDocument,
StorageDocument firstDocument,
CancellationToken cancellationToken)
{
if (firstDocument.PayloadId is null)

20
src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Integration/EwsVerdictDeterminismTests.cs
View File

@@ -52,13 +52,13 @@ public sealed class EwsVerdictDeterminismTests
// Assert - All results should be byte-identical
var firstScore = results[0].Score;
var firstBucket = results[0].Bucket;
var firstDimensions = results[0].Dimensions;
var firstBreakdown = results[0].Breakdown;
results.Should().AllSatisfy(r =>
{
r.Score.Should().Be(firstScore, "score must be deterministic");
r.Bucket.Should().Be(firstBucket, "bucket must be deterministic");
r.Dimensions.Should().BeEquivalentTo(firstDimensions, "dimensions must be deterministic");
r.Breakdown.Should().BeEquivalentTo(firstBreakdown, "breakdown must be deterministic");
});
}
@@ -155,16 +155,16 @@ public sealed class EwsVerdictDeterminismTests
// Custom policy with different weights
var customPolicy = new EvidenceWeightPolicy
{
PolicyId = "custom-test-policy",
Version = "1.0",
Version = "ews.v1",
Profile = "custom-test-policy",
Weights = new EvidenceWeights
{
Reachability = 0.50, // Much higher weight on reachability
Runtime = 0.10,
Backport = 0.05,
Exploit = 0.20,
Source = 0.10,
Mitigation = 0.05
Rch = 0.50, // Much higher weight on reachability
Rts = 0.10,
Bkp = 0.05,
Xpl = 0.20,
Src = 0.10,
Mit = 0.05
},
Buckets = EvidenceWeightPolicy.DefaultProduction.Buckets
};

8
src/__Libraries/__Tests/StellaOps.Provcache.Tests/RevocationLedgerTests.cs
View File

@@ -189,17 +189,17 @@ public sealed class RevocationLedgerTests
}
[Fact]
public void Clear_RemovesAllEntries()
public async Task Clear_RemovesAllEntries()
{
// Arrange
_ledger.RecordAsync(CreateTestEntry(RevocationTypes.Signer, "s1")).GetAwaiter().GetResult();
_ledger.RecordAsync(CreateTestEntry(RevocationTypes.Signer, "s2")).GetAwaiter().GetResult();
await _ledger.RecordAsync(CreateTestEntry(RevocationTypes.Signer, "s1"));
await _ledger.RecordAsync(CreateTestEntry(RevocationTypes.Signer, "s2"));
// Act
_ledger.Clear();
// Assert
var seqNo = _ledger.GetLatestSeqNoAsync().GetAwaiter().GetResult();
var seqNo = await _ledger.GetLatestSeqNoAsync();
seqNo.Should().Be(0);
}