stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

Add tests and implement timeline ingestion options with NATS and Redis subscribers

Browse Source 
- Introduced `BinaryReachabilityLifterTests` to validate binary lifting functionality.
- Created `PackRunWorkerOptions` for configuring worker paths and execution persistence.
- Added `TimelineIngestionOptions` for configuring NATS and Redis ingestion transports.
- Implemented `NatsTimelineEventSubscriber` for subscribing to NATS events.
- Developed `RedisTimelineEventSubscriber` for reading from Redis Streams.
- Added `TimelineEnvelopeParser` to normalize incoming event envelopes.
- Created unit tests for `TimelineEnvelopeParser` to ensure correct field mapping.
- Implemented `TimelineAuthorizationAuditSink` for logging authorization outcomes.
This commit is contained in:
StellaOps Bot
2025-12-03 09:46:48 +02:00
parent e923880694
commit 35c8f9216f
520 changed files with 4416 additions and 31492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
src/ExportCenter/StellaOps.ExportCenter.RiskBundles/RiskBundleBuilder.cs
View File

@@ -24,7 +24,7 @@ public sealed class RiskBundleBuilder
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public RiskBundleBuildResult Build(RiskBundleBuildRequest request, CancellationToken cancellationToken = default)
public RiskBundlePreparedBuild Prepare(RiskBundleBuildRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
if (request.BundleId == Guid.Empty)
@@ -42,19 +42,35 @@ public sealed class RiskBundleBuilder
{
throw new InvalidOperationException("IncludeOsv was requested but no OSV provider input was supplied.");
}
var manifest = BuildManifest(request.BundleId, providers);
var manifestJson = JsonSerializer.Serialize(manifest, SerializerOptions);
var rootHash = ComputeSha256(manifestJson);
return new RiskBundlePreparedBuild(manifest, manifestJson, rootHash, providers);
}
public RiskBundleBuildResult Build(RiskBundleBuildRequest request, CancellationToken cancellationToken = default)
=> Build(request, additionalFiles: null, prepared: null, cancellationToken);
public RiskBundleBuildResult Build(
RiskBundleBuildRequest request,
IEnumerable<RiskBundleAdditionalFile>? additionalFiles,
RiskBundlePreparedBuild? prepared = null,
CancellationToken cancellationToken = default)
{
var preparedBuild = prepared ?? Prepare(request, cancellationToken);
var bundleStream = CreateBundleStream(
providers,
preparedBuild.Providers,
request.ManifestFileName,
manifestJson,
request.BundleFileName ?? "risk-bundle.tar.gz");
preparedBuild.ManifestJson,
request.BundleFileName ?? "risk-bundle.tar.gz",
additionalFiles);
bundleStream.Position = 0;
return new RiskBundleBuildResult(manifest, manifestJson, rootHash, bundleStream);
return new RiskBundleBuildResult(preparedBuild.Manifest, preparedBuild.ManifestJson, preparedBuild.RootHash, bundleStream);
}
private static List<RiskBundleProviderEntry> CollectProviders(RiskBundleBuildRequest request, CancellationToken cancellationToken)
@@ -210,7 +226,8 @@ public sealed class RiskBundleBuilder
IReadOnlyList<RiskBundleProviderEntry> providers,
string manifestFileName,
string manifestJson,
string bundleFileName)
string bundleFileName,
IEnumerable<RiskBundleAdditionalFile>? additionalFiles)
{
var stream = new MemoryStream();
using (var gzip = new GZipStream(stream, CompressionLevel.SmallestSize, leaveOpen: true))
@@ -222,6 +239,14 @@ public sealed class RiskBundleBuilder
{
WriteProviderFile(tar, provider);
}
if (additionalFiles is not null)
{
foreach (var file in additionalFiles)
{
WriteBytesEntry(tar, file.BundlePath, file.Content, file.Mode);
}
}
}
ApplyDeterministicGzipHeader(stream);
@@ -267,6 +292,18 @@ public sealed class RiskBundleBuilder
writer.WriteEntry(entry);
}
private static void WriteBytesEntry(TarWriter writer, string path, byte[] content, UnixFileMode mode)
{
using var dataStream = new MemoryStream(content);
var entry = new PaxTarEntry(TarEntryType.RegularFile, path)
{
Mode = mode,
ModificationTime = FixedTimestamp,
DataStream = dataStream
};
writer.WriteEntry(entry);
}
private static void ApplyDeterministicGzipHeader(MemoryStream stream)
{
if (stream.Length < 10)
@@ -295,3 +332,28 @@ public sealed class RiskBundleBuilder
return File.Exists(full) ? full : null;
}
}
public sealed record RiskBundlePreparedBuild(
RiskBundleManifest Manifest,
string ManifestJson,
string RootHash,
IReadOnlyList<RiskBundleProviderEntry> Providers);
public sealed record RiskBundleAdditionalFile(
string BundlePath,
byte[] Content,
UnixFileMode Mode)
{
public static RiskBundleAdditionalFile FromText(string bundlePath, string content, UnixFileMode? mode = null)
{
if (string.IsNullOrWhiteSpace(bundlePath))
{
throw new ArgumentException("Bundle path is required for additional file entries.", nameof(bundlePath));
}
var data = Encoding.UTF8.GetBytes(content ?? string.Empty);
return new RiskBundleAdditionalFile(bundlePath, data, mode ?? DefaultFileModeValue);
}
public const UnixFileMode DefaultFileModeValue = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead;
}

57
src/ExportCenter/StellaOps.ExportCenter.RiskBundles/RiskBundleJob.cs
View File

@@ -1,3 +1,6 @@
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
namespace StellaOps.ExportCenter.RiskBundles;
@@ -14,25 +17,30 @@ public sealed record RiskBundleJobOutcome(
RiskBundleStorageMetadata ManifestStorage,
RiskBundleStorageMetadata ManifestSignatureStorage,
RiskBundleStorageMetadata BundleStorage,
RiskBundleStorageMetadata BundleSignatureStorage,
string ManifestJson,
string ManifestSignatureJson,
string BundleSignature,
string RootHash);
public sealed class RiskBundleJob
{
private readonly RiskBundleBuilder _builder;
private readonly IRiskBundleManifestSigner _signer;
private readonly IRiskBundleArchiveSigner _archiveSigner;
private readonly IRiskBundleObjectStore _objectStore;
private readonly ILogger<RiskBundleJob> _logger;
public RiskBundleJob(
RiskBundleBuilder builder,
IRiskBundleManifestSigner signer,
IRiskBundleArchiveSigner archiveSigner,
IRiskBundleObjectStore objectStore,
ILogger<RiskBundleJob> logger)
{
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
_signer = signer ?? throw new ArgumentNullException(nameof(signer));
_archiveSigner = archiveSigner ?? throw new ArgumentNullException(nameof(archiveSigner));
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -42,43 +50,61 @@ public sealed class RiskBundleJob
ArgumentNullException.ThrowIfNull(request);
cancellationToken.ThrowIfCancellationRequested();
var build = _builder.Build(request.BuildRequest, cancellationToken);
_logger.LogInformation("Risk bundle built with {ProviderCount} providers.", build.Manifest.Providers.Count);
var bundleFileName = string.IsNullOrWhiteSpace(request.BundleFileName)
? "risk-bundle.tar.gz"
: request.BundleFileName;
var signature = await _signer.SignAsync(build.ManifestJson, cancellationToken).ConfigureAwait(false);
var signatureJson = System.Text.Json.JsonSerializer.Serialize(signature, new System.Text.Json.JsonSerializerOptions
var prepared = _builder.Prepare(request.BuildRequest, cancellationToken);
_logger.LogInformation("Risk bundle built with {ProviderCount} providers (prepared).", prepared.Manifest.Providers.Count);
var manifestSignature = await _signer.SignAsync(prepared.ManifestJson, cancellationToken).ConfigureAwait(false);
var signatureJson = JsonSerializer.Serialize(manifestSignature, SerializerOptions);
var additionalFiles = new[]
{
PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false
});
RiskBundleAdditionalFile.FromText($"signatures/{request.BuildRequest.ManifestDsseFileName}", signatureJson)
};
var build = _builder.Build(request.BuildRequest, additionalFiles, prepared, cancellationToken);
var bundleSignature = await _archiveSigner.SignArchiveAsync(build.BundleStream, cancellationToken).ConfigureAwait(false);
var bundleSignatureFileName = $"{bundleFileName}.sig";
var manifestKey = Combine(request.StoragePrefix, request.BuildRequest.ManifestFileName);
var manifestSigKey = Combine(request.StoragePrefix, request.BuildRequest.ManifestDsseFileName);
var bundleKey = Combine(request.StoragePrefix, request.BundleFileName);
var manifestSigKey = Combine(request.StoragePrefix, $"signatures/{request.BuildRequest.ManifestDsseFileName}");
var bundleKey = Combine(request.StoragePrefix, bundleFileName);
var bundleSigKey = Combine(request.StoragePrefix, $"signatures/{bundleSignatureFileName}");
var manifestStorage = await _objectStore.StoreAsync(
new RiskBundleObjectStoreOptions(manifestKey, "application/json"),
new MemoryStream(System.Text.Encoding.UTF8.GetBytes(build.ManifestJson)),
new MemoryStream(Encoding.UTF8.GetBytes(build.ManifestJson)),
cancellationToken).ConfigureAwait(false);
var signatureStorage = await _objectStore.StoreAsync(
new RiskBundleObjectStoreOptions(manifestSigKey, "application/json"),
new MemoryStream(System.Text.Encoding.UTF8.GetBytes(signatureJson)),
new MemoryStream(Encoding.UTF8.GetBytes(signatureJson)),
cancellationToken).ConfigureAwait(false);
build.BundleStream.Position = 0;
var bundleStorage = await _objectStore.StoreAsync(
new RiskBundleObjectStoreOptions(bundleKey, "application/gzip"),
build.BundleStream,
cancellationToken).ConfigureAwait(false);
var bundleSignatureStorage = await _objectStore.StoreAsync(
new RiskBundleObjectStoreOptions(bundleSigKey, "application/vnd.stellaops.signature"),
new MemoryStream(Encoding.UTF8.GetBytes(bundleSignature)),
cancellationToken).ConfigureAwait(false);
return new RiskBundleJobOutcome(
build.Manifest,
manifestStorage,
signatureStorage,
bundleStorage,
bundleSignatureStorage,
build.ManifestJson,
signatureJson,
bundleSignature,
build.RootHash);
}
@@ -94,4 +120,11 @@ public sealed class RiskBundleJob
? fileName
: $"{sanitizedPrefix}/{fileName}";
}
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false
};
}

25
src/ExportCenter/StellaOps.ExportCenter.RiskBundles/RiskBundleSigning.cs
View File

@@ -1,4 +1,5 @@
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json.Serialization;
@@ -10,6 +11,11 @@ public interface IRiskBundleManifestSigner
Task<RiskBundleManifestSignatureDocument> SignAsync(string manifestJson, CancellationToken cancellationToken = default);
}
public interface IRiskBundleArchiveSigner
{
Task<string> SignArchiveAsync(Stream archiveStream, CancellationToken cancellationToken = default);
}
public sealed record RiskBundleManifestSignatureDocument(
[property: JsonPropertyName("payloadType")] string PayloadType,
[property: JsonPropertyName("payload")] string Payload,
@@ -19,7 +25,7 @@ public sealed record RiskBundleManifestDsseSignature(
[property: JsonPropertyName("sig")] string Signature,
[property: JsonPropertyName("keyid")] string KeyId);
public sealed class HmacRiskBundleManifestSigner : IRiskBundleManifestSigner
public sealed class HmacRiskBundleManifestSigner : IRiskBundleManifestSigner, IRiskBundleArchiveSigner
{
private const string DefaultPayloadType = "application/stellaops.risk-bundle.provider-manifest+json";
private readonly byte[] _key;
@@ -52,6 +58,23 @@ public sealed class HmacRiskBundleManifestSigner : IRiskBundleManifestSigner
return Task.FromResult(document);
}
public Task<string> SignArchiveAsync(Stream archiveStream, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(archiveStream);
cancellationToken.ThrowIfCancellationRequested();
if (!archiveStream.CanSeek)
{
throw new ArgumentException("Archive stream must support seeking for signing.", nameof(archiveStream));
}
archiveStream.Position = 0;
using var hmac = new HMACSHA256(_key);
var signature = hmac.ComputeHash(archiveStream);
archiveStream.Position = 0;
return Task.FromResult(Convert.ToBase64String(signature));
}
private static string ComputeHmac(byte[] pae, byte[] key)
{
using var hmac = new HMACSHA256(key);

34
src/ExportCenter/StellaOps.ExportCenter/StellaOps.ExportCenter.Tests/RiskBundleJobTests.cs
View File

@@ -1,3 +1,7 @@
using System.Formats.Tar;
using System.IO.Compression;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.ExportCenter.RiskBundles;
@@ -15,19 +19,41 @@ public sealed class RiskBundleJobTests
Guid.NewGuid(),
Providers: new[] { new RiskBundleProviderInput("cisa-kev", providerPath, "CISA KEV") });
var signer = new HmacRiskBundleManifestSigner("secret", "risk-key");
var store = new InMemoryObjectStore();
var job = new RiskBundleJob(
new RiskBundleBuilder(),
new HmacRiskBundleManifestSigner("secret", "risk-key"),
new InMemoryObjectStore(),
signer,
signer,
store,
NullLogger<RiskBundleJob>.Instance);
var outcome = await job.ExecuteAsync(new RiskBundleJobRequest(buildRequest), TestContext.Current.CancellationToken);
Assert.Equal("risk-bundles/provider-manifest.json", outcome.ManifestStorage.StorageKey);
Assert.Equal("risk-bundles/provider-manifest.dsse", outcome.ManifestSignatureStorage.StorageKey);
Assert.Equal("risk-bundles/signatures/provider-manifest.dsse", outcome.ManifestSignatureStorage.StorageKey);
Assert.Equal("risk-bundles/risk-bundle.tar.gz", outcome.BundleStorage.StorageKey);
Assert.Equal("risk-bundles/signatures/risk-bundle.tar.gz.sig", outcome.BundleSignatureStorage.StorageKey);
Assert.False(string.IsNullOrWhiteSpace(outcome.ManifestJson));
Assert.False(string.IsNullOrWhiteSpace(outcome.ManifestSignatureJson));
Assert.False(string.IsNullOrWhiteSpace(outcome.BundleSignature));
using var gzip = new GZipStream(new MemoryStream(store.Get(outcome.BundleStorage.StorageKey)), CompressionMode.Decompress, leaveOpen: false);
using var tar = new TarReader(gzip, leaveOpen: false);
var entries = new List<string>();
TarEntry? entry;
while ((entry = tar.GetNextEntry()) is not null)
{
entries.Add(entry.Name);
}
Assert.Contains("signatures/provider-manifest.dsse", entries);
var dsseDocument = JsonSerializer.Deserialize<RiskBundleManifestSignatureDocument>(Encoding.UTF8.GetString(store.Get(outcome.ManifestSignatureStorage.StorageKey)))!;
Assert.Equal("application/stellaops.risk-bundle.provider-manifest+json", dsseDocument.PayloadType);
var bundleSignature = Encoding.UTF8.GetString(store.Get(outcome.BundleSignatureStorage.StorageKey));
Assert.Equal(outcome.BundleSignature, bundleSignature);
}
private sealed class InMemoryObjectStore : IRiskBundleObjectStore
@@ -41,6 +67,8 @@ public sealed class RiskBundleJobTests
_store[options.StorageKey] = ms.ToArray();
return Task.FromResult(new RiskBundleStorageMetadata(options.StorageKey, ms.Length, options.ContentType));
}
public byte[] Get(string key) => _store[key];
}
private sealed class TempDir : IDisposable

1
src/ExportCenter/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/Program.cs
View File

@@ -32,6 +32,7 @@ builder.Services.AddSingleton<IRiskBundleManifestSigner>(sp =>
var keyId = string.IsNullOrWhiteSpace(signing.KeyId) ? "risk-bundle-hmac" : signing.KeyId!;
return new HmacRiskBundleManifestSigner(key, keyId);
});
builder.Services.AddSingleton<IRiskBundleArchiveSigner>(sp => (IRiskBundleArchiveSigner)sp.GetRequiredService<IRiskBundleManifestSigner>());
builder.Services.AddSingleton<IRiskBundleObjectStore, FileSystemRiskBundleObjectStore>();
builder.Services.AddSingleton<RiskBundleJob>();

8
src/ExportCenter/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/RiskBundleWorker.cs
View File

@@ -71,11 +71,15 @@ public sealed class RiskBundleWorker : BackgroundService
BundlePrefix: options.StoragePrefix ?? "risk-bundles",
ManifestFileName: options.ManifestFileName ?? "provider-manifest.json",
ManifestDsseFileName: options.ManifestDsseFileName ?? "provider-manifest.dsse",
AllowMissingOptional: options.AllowMissingOptional);
AllowMissingOptional: options.AllowMissingOptional,
AllowStaleOptional: options.AllowStaleOptional,
IncludeOsv: options.IncludeOsv);
return new RiskBundleJobRequest(
build,
StoragePrefix: options.StoragePrefix ?? "risk-bundles",
BundleFileName: options.BundleFileName ?? "risk-bundle.tar.gz");
BundleFileName: options.BundleFileName ?? "risk-bundle.tar.gz",
IncludeOsv: options.IncludeOsv,
AllowStaleOptional: options.AllowStaleOptional);
}
}

10
src/ExportCenter/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/RiskBundleWorkerOptions.cs
View File

@@ -19,6 +19,10 @@ public sealed class RiskBundleWorkerOptions
public bool AllowMissingOptional { get; set; } = true;
public bool AllowStaleOptional { get; set; } = true;
public bool IncludeOsv { get; set; } = false;
[MinLength(1)]
public List<RiskBundleProviderOption> Providers { get; set; } = new();
}
@@ -28,6 +32,7 @@ public sealed class RiskBundleProviderOption
public string? ProviderId { get; set; }
public string? SourcePath { get; set; }
public string? Source { get; set; }
public string? SignaturePath { get; set; }
public bool Optional { get; set; }
public DateOnly? SnapshotDate { get; set; }
@@ -52,8 +57,9 @@ public sealed class RiskBundleProviderOption
ProviderId,
SourcePath,
Source,
Optional,
SnapshotDate);
SignaturePath: SignaturePath,
Optional: Optional,
SnapshotDate: SnapshotDate);
}
}

2
src/ExportCenter/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/appsettings.Development.json
View File

@@ -7,6 +7,8 @@
},
"RiskBundles": {
"Enabled": false,
"AllowStaleOptional": true,
"IncludeOsv": false,
"Storage": {
"RootPath": "./out/risk-bundles-dev"
},

2
src/ExportCenter/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/appsettings.json
View File

@@ -12,6 +12,8 @@
"ManifestFileName": "provider-manifest.json",
"ManifestDsseFileName": "provider-manifest.dsse",
"AllowMissingOptional": true,
"AllowStaleOptional": true,
"IncludeOsv": false,
"Storage": {
"RootPath": "./out/risk-bundles"
},