Add tests and implement timeline ingestion options with NATS and Redis subscribers

- Introduced `BinaryReachabilityLifterTests` to validate binary lifting functionality.
- Created `PackRunWorkerOptions` for configuring worker paths and execution persistence.
- Added `TimelineIngestionOptions` for configuring NATS and Redis ingestion transports.
- Implemented `NatsTimelineEventSubscriber` for subscribing to NATS events.
- Developed `RedisTimelineEventSubscriber` for reading from Redis Streams.
- Added `TimelineEnvelopeParser` to normalize incoming event envelopes.
- Created unit tests for `TimelineEnvelopeParser` to ensure correct field mapping.
- Implemented `TimelineAuthorizationAuditSink` for logging authorization outcomes.
This commit is contained in:
StellaOps Bot
2025-12-03 09:46:48 +02:00
parent e923880694
commit 35c8f9216f
520 changed files with 4416 additions and 31492 deletions

View File

@@ -1,3 +1,7 @@
using System.Formats.Tar;
using System.IO.Compression;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.ExportCenter.RiskBundles;
@@ -15,19 +19,41 @@ public sealed class RiskBundleJobTests
Guid.NewGuid(),
Providers: new[] { new RiskBundleProviderInput("cisa-kev", providerPath, "CISA KEV") });
var signer = new HmacRiskBundleManifestSigner("secret", "risk-key");
var store = new InMemoryObjectStore();
var job = new RiskBundleJob(
new RiskBundleBuilder(),
new HmacRiskBundleManifestSigner("secret", "risk-key"),
new InMemoryObjectStore(),
signer,
signer,
store,
NullLogger<RiskBundleJob>.Instance);
var outcome = await job.ExecuteAsync(new RiskBundleJobRequest(buildRequest), TestContext.Current.CancellationToken);
Assert.Equal("risk-bundles/provider-manifest.json", outcome.ManifestStorage.StorageKey);
Assert.Equal("risk-bundles/provider-manifest.dsse", outcome.ManifestSignatureStorage.StorageKey);
Assert.Equal("risk-bundles/signatures/provider-manifest.dsse", outcome.ManifestSignatureStorage.StorageKey);
Assert.Equal("risk-bundles/risk-bundle.tar.gz", outcome.BundleStorage.StorageKey);
Assert.Equal("risk-bundles/signatures/risk-bundle.tar.gz.sig", outcome.BundleSignatureStorage.StorageKey);
Assert.False(string.IsNullOrWhiteSpace(outcome.ManifestJson));
Assert.False(string.IsNullOrWhiteSpace(outcome.ManifestSignatureJson));
Assert.False(string.IsNullOrWhiteSpace(outcome.BundleSignature));
using var gzip = new GZipStream(new MemoryStream(store.Get(outcome.BundleStorage.StorageKey)), CompressionMode.Decompress, leaveOpen: false);
using var tar = new TarReader(gzip, leaveOpen: false);
var entries = new List<string>();
TarEntry? entry;
while ((entry = tar.GetNextEntry()) is not null)
{
entries.Add(entry.Name);
}
Assert.Contains("signatures/provider-manifest.dsse", entries);
var dsseDocument = JsonSerializer.Deserialize<RiskBundleManifestSignatureDocument>(Encoding.UTF8.GetString(store.Get(outcome.ManifestSignatureStorage.StorageKey)))!;
Assert.Equal("application/stellaops.risk-bundle.provider-manifest+json", dsseDocument.PayloadType);
var bundleSignature = Encoding.UTF8.GetString(store.Get(outcome.BundleSignatureStorage.StorageKey));
Assert.Equal(outcome.BundleSignature, bundleSignature);
}
private sealed class InMemoryObjectStore : IRiskBundleObjectStore
@@ -41,6 +67,8 @@ public sealed class RiskBundleJobTests
_store[options.StorageKey] = ms.ToArray();
return Task.FromResult(new RiskBundleStorageMetadata(options.StorageKey, ms.Length, options.ContentType));
}
public byte[] Get(string key) => _store[key];
}
private sealed class TempDir : IDisposable

View File

@@ -32,6 +32,7 @@ builder.Services.AddSingleton<IRiskBundleManifestSigner>(sp =>
var keyId = string.IsNullOrWhiteSpace(signing.KeyId) ? "risk-bundle-hmac" : signing.KeyId!;
return new HmacRiskBundleManifestSigner(key, keyId);
});
builder.Services.AddSingleton<IRiskBundleArchiveSigner>(sp => (IRiskBundleArchiveSigner)sp.GetRequiredService<IRiskBundleManifestSigner>());
builder.Services.AddSingleton<IRiskBundleObjectStore, FileSystemRiskBundleObjectStore>();
builder.Services.AddSingleton<RiskBundleJob>();

View File

@@ -71,11 +71,15 @@ public sealed class RiskBundleWorker : BackgroundService
BundlePrefix: options.StoragePrefix ?? "risk-bundles",
ManifestFileName: options.ManifestFileName ?? "provider-manifest.json",
ManifestDsseFileName: options.ManifestDsseFileName ?? "provider-manifest.dsse",
AllowMissingOptional: options.AllowMissingOptional);
AllowMissingOptional: options.AllowMissingOptional,
AllowStaleOptional: options.AllowStaleOptional,
IncludeOsv: options.IncludeOsv);
return new RiskBundleJobRequest(
build,
StoragePrefix: options.StoragePrefix ?? "risk-bundles",
BundleFileName: options.BundleFileName ?? "risk-bundle.tar.gz");
BundleFileName: options.BundleFileName ?? "risk-bundle.tar.gz",
IncludeOsv: options.IncludeOsv,
AllowStaleOptional: options.AllowStaleOptional);
}
}

View File

@@ -19,6 +19,10 @@ public sealed class RiskBundleWorkerOptions
public bool AllowMissingOptional { get; set; } = true;
public bool AllowStaleOptional { get; set; } = true;
public bool IncludeOsv { get; set; } = false;
[MinLength(1)]
public List<RiskBundleProviderOption> Providers { get; set; } = new();
}
@@ -28,6 +32,7 @@ public sealed class RiskBundleProviderOption
public string? ProviderId { get; set; }
public string? SourcePath { get; set; }
public string? Source { get; set; }
public string? SignaturePath { get; set; }
public bool Optional { get; set; }
public DateOnly? SnapshotDate { get; set; }
@@ -52,8 +57,9 @@ public sealed class RiskBundleProviderOption
ProviderId,
SourcePath,
Source,
Optional,
SnapshotDate);
SignaturePath: SignaturePath,
Optional: Optional,
SnapshotDate: SnapshotDate);
}
}

View File

@@ -7,6 +7,8 @@
},
"RiskBundles": {
"Enabled": false,
"AllowStaleOptional": true,
"IncludeOsv": false,
"Storage": {
"RootPath": "./out/risk-bundles-dev"
},

View File

@@ -12,6 +12,8 @@
"ManifestFileName": "provider-manifest.json",
"ManifestDsseFileName": "provider-manifest.dsse",
"AllowMissingOptional": true,
"AllowStaleOptional": true,
"IncludeOsv": false,
"Storage": {
"RootPath": "./out/risk-bundles"
},