feat: Add Scanner CI runner and related artifacts
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Airgap Sealed CI Smoke / sealed-smoke (push) Has been cancelled
Export Center CI / export-ci (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled

- Implemented `run-scanner-ci.sh` to build and run tests for the Scanner solution with a warmed NuGet cache.
- Created `excititor-vex-traces.json` dashboard for monitoring Excititor VEX observations.
- Added Docker Compose configuration for the OTLP span sink in `docker-compose.spansink.yml`.
- Configured OpenTelemetry collector in `otel-spansink.yaml` to receive and process traces.
- Developed `run-spansink.sh` script to run the OTLP span sink for Excititor traces.
- Introduced `FileSystemRiskBundleObjectStore` for storing risk bundle artifacts in the filesystem.
- Built `RiskBundleBuilder` for creating risk bundles with associated metadata and providers.
- Established `RiskBundleJob` to execute the risk bundle creation and storage process.
- Defined models for risk bundle inputs, entries, and manifests in `RiskBundleModels.cs`.
- Implemented signing functionality for risk bundle manifests with `HmacRiskBundleManifestSigner`.
- Created unit tests for `RiskBundleBuilder`, `RiskBundleJob`, and signing functionality to ensure correctness.
- Added filesystem artifact reader tests to validate manifest parsing and artifact listing.
- Included test manifests for egress scenarios in the task runner tests.
- Developed timeline query service tests to verify tenant and event ID handling.
This commit is contained in:
StellaOps Bot
2025-11-30 19:12:35 +02:00
parent 17d45a6d30
commit 71e9a56cfd
92 changed files with 2596 additions and 387 deletions

View File

@@ -0,0 +1,55 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.ExportCenter.RiskBundles;
public sealed class FileSystemRiskBundleObjectStore : IRiskBundleObjectStore
{
private readonly IOptionsMonitor<FileSystemRiskBundleStorageOptions> _options;
private readonly ILogger<FileSystemRiskBundleObjectStore> _logger;
public FileSystemRiskBundleObjectStore(
IOptionsMonitor<FileSystemRiskBundleStorageOptions> options,
ILogger<FileSystemRiskBundleObjectStore> logger)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RiskBundleStorageMetadata> StoreAsync(
RiskBundleObjectStoreOptions options,
Stream content,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(content);
var root = _options.CurrentValue.RootPath;
if (string.IsNullOrWhiteSpace(root))
{
throw new InvalidOperationException("Risk bundle storage root path is not configured.");
}
var fullPath = Path.Combine(root, options.StorageKey);
var directory = Path.GetDirectoryName(fullPath);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
}
await using (var file = new FileStream(fullPath, FileMode.Create, FileAccess.Write, FileShare.None, 128 * 1024, FileOptions.Asynchronous | FileOptions.WriteThrough))
{
await content.CopyToAsync(file, 128 * 1024, cancellationToken).ConfigureAwait(false);
}
var length = new FileInfo(fullPath).Length;
_logger.LogInformation("Risk bundle artefact stored at {Path} ({Bytes} bytes).", fullPath, length);
return new RiskBundleStorageMetadata(options.StorageKey, length, options.ContentType);
}
}
public sealed class FileSystemRiskBundleStorageOptions
{
public string? RootPath { get; set; }
}

View File

@@ -0,0 +1,241 @@
using System.Buffers.Binary;
using System.Collections.Immutable;
using System.Formats.Tar;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.ExportCenter.RiskBundles;
public sealed class RiskBundleBuilder
{
private const string ManifestVersion = "1";
private static readonly UnixFileMode DefaultFileMode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead;
private static readonly DateTimeOffset FixedTimestamp = new(2024, 01, 01, 0, 0, 0, TimeSpan.Zero);
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public RiskBundleBuildResult Build(RiskBundleBuildRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
if (request.BundleId == Guid.Empty)
{
throw new ArgumentException("Bundle identifier must be provided.", nameof(request));
}
if (request.Providers is not { Count: > 0 })
{
throw new ArgumentException("At least one provider input is required.", nameof(request));
}
var providers = CollectProviders(request, cancellationToken);
var manifest = BuildManifest(request.BundleId, providers);
var manifestJson = JsonSerializer.Serialize(manifest, SerializerOptions);
var rootHash = ComputeSha256(manifestJson);
var bundleStream = CreateBundleStream(
providers,
request.ManifestFileName,
manifestJson,
request.BundleFileName ?? "risk-bundle.tar.gz");
bundleStream.Position = 0;
return new RiskBundleBuildResult(manifest, manifestJson, rootHash, bundleStream);
}
private static List<RiskBundleProviderEntry> CollectProviders(RiskBundleBuildRequest request, CancellationToken cancellationToken)
{
var entries = new List<RiskBundleProviderEntry>(request.Providers.Count);
foreach (var provider in request.Providers.OrderBy(p => p.ProviderId, StringComparer.Ordinal))
{
cancellationToken.ThrowIfCancellationRequested();
if (provider is null)
{
throw new ArgumentException("Provider list cannot contain null entries.", nameof(request));
}
if (string.IsNullOrWhiteSpace(provider.ProviderId))
{
throw new ArgumentException("ProviderId cannot be empty.", nameof(provider.ProviderId));
}
if (string.IsNullOrWhiteSpace(provider.SourcePath))
{
throw new ArgumentException("SourcePath cannot be empty.", nameof(provider.SourcePath));
}
var fullPath = Path.GetFullPath(provider.SourcePath);
if (!File.Exists(fullPath))
{
if (provider.Optional && request.AllowMissingOptional)
{
continue;
}
throw new FileNotFoundException($"Provider source file '{fullPath}' not found.", fullPath);
}
var sha256 = ComputeSha256FromFile(fullPath);
var size = new FileInfo(fullPath).Length;
var bundlePath = $"providers/{provider.ProviderId}/snapshot";
entries.Add(new RiskBundleProviderEntry(
provider.ProviderId,
provider.Source,
provider.SnapshotDate,
sha256,
size,
provider.Optional,
bundlePath,
fullPath,
SignaturePath: null));
}
if (entries.Count == 0)
{
throw new InvalidOperationException("No provider artefacts collected. Provide at least one valid provider input.");
}
return entries;
}
private static RiskBundleManifest BuildManifest(Guid bundleId, List<RiskBundleProviderEntry> providers)
{
var inputsHash = ComputeInputsHash(providers);
return new RiskBundleManifest(
ManifestVersion,
bundleId,
FixedTimestamp,
providers.ToImmutableArray(),
inputsHash);
}
private static string ComputeInputsHash(IEnumerable<RiskBundleProviderEntry> providers)
{
using var sha = SHA256.Create();
var builder = new StringBuilder();
foreach (var entry in providers.OrderBy(e => e.ProviderId, StringComparer.Ordinal))
{
builder.Append(entry.ProviderId)
.Append('\0')
.Append(entry.Sha256)
.Append('\0')
.Append(entry.Source)
.Append('\0')
.Append(entry.Optional ? '1' : '0')
.Append('\0')
.Append(entry.SnapshotDate?.ToString("yyyy-MM-dd") ?? string.Empty)
.Append('\0');
}
var bytes = Encoding.UTF8.GetBytes(builder.ToString());
var hash = sha.ComputeHash(bytes);
return ToHex(hash);
}
private static string ComputeSha256(string content)
{
var bytes = Encoding.UTF8.GetBytes(content);
using var sha = SHA256.Create();
var hash = sha.ComputeHash(bytes);
return ToHex(hash);
}
private static string ComputeSha256FromFile(string filePath)
{
using var stream = File.OpenRead(filePath);
using var sha = SHA256.Create();
var hash = sha.ComputeHash(stream);
return ToHex(hash);
}
private static string ToHex(ReadOnlySpan<byte> bytes)
{
Span<byte> hex = stackalloc byte[bytes.Length * 2];
for (var i = 0; i < bytes.Length; i++)
{
var b = bytes[i];
hex[i * 2] = GetHexValue(b / 16);
hex[i * 2 + 1] = GetHexValue(b % 16);
}
return Encoding.ASCII.GetString(hex);
}
private static byte GetHexValue(int i) => (byte)(i < 10 ? i + 48 : i - 10 + 97);
private static MemoryStream CreateBundleStream(
IReadOnlyList<RiskBundleProviderEntry> providers,
string manifestFileName,
string manifestJson,
string bundleFileName)
{
var stream = new MemoryStream();
using (var gzip = new GZipStream(stream, CompressionLevel.SmallestSize, leaveOpen: true))
using (var tar = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: true))
{
WriteTextEntry(tar, $"manifests/{manifestFileName}", manifestJson, DefaultFileMode);
foreach (var provider in providers)
{
WriteProviderFile(tar, provider);
}
}
ApplyDeterministicGzipHeader(stream);
stream.Position = 0;
return stream;
}
private static void WriteProviderFile(TarWriter writer, RiskBundleProviderEntry entry)
{
var filePath = entry.SourceFilePath ?? throw new InvalidOperationException("Source file path missing for provider entry.");
using var dataStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 128 * 1024, FileOptions.SequentialScan);
var tarEntry = new PaxTarEntry(TarEntryType.RegularFile, entry.BundlePath)
{
Mode = DefaultFileMode,
ModificationTime = FixedTimestamp,
DataStream = dataStream
};
writer.WriteEntry(tarEntry);
}
private static void WriteTextEntry(TarWriter writer, string path, string content, UnixFileMode mode)
{
var bytes = Encoding.UTF8.GetBytes(content);
using var dataStream = new MemoryStream(bytes);
var entry = new PaxTarEntry(TarEntryType.RegularFile, path)
{
Mode = mode,
ModificationTime = FixedTimestamp,
DataStream = dataStream
};
writer.WriteEntry(entry);
}
private static void ApplyDeterministicGzipHeader(MemoryStream stream)
{
if (stream.Length < 10)
{
throw new InvalidOperationException("GZip header not fully written for risk bundle.");
}
var seconds = checked((int)(FixedTimestamp - DateTimeOffset.UnixEpoch).TotalSeconds);
Span<byte> buffer = stackalloc byte[4];
BinaryPrimitives.WriteInt32LittleEndian(buffer, seconds);
var originalPosition = stream.Position;
stream.Position = 4;
stream.Write(buffer);
stream.Position = originalPosition;
}
}

View File

@@ -0,0 +1,95 @@
using Microsoft.Extensions.Logging;
namespace StellaOps.ExportCenter.RiskBundles;
public sealed record RiskBundleJobRequest(
RiskBundleBuildRequest BuildRequest,
string StoragePrefix = "risk-bundles",
string BundleFileName = "risk-bundle.tar.gz");
public sealed record RiskBundleJobOutcome(
RiskBundleManifest Manifest,
RiskBundleStorageMetadata ManifestStorage,
RiskBundleStorageMetadata ManifestSignatureStorage,
RiskBundleStorageMetadata BundleStorage,
string ManifestJson,
string ManifestSignatureJson,
string RootHash);
public sealed class RiskBundleJob
{
private readonly RiskBundleBuilder _builder;
private readonly IRiskBundleManifestSigner _signer;
private readonly IRiskBundleObjectStore _objectStore;
private readonly ILogger<RiskBundleJob> _logger;
public RiskBundleJob(
RiskBundleBuilder builder,
IRiskBundleManifestSigner signer,
IRiskBundleObjectStore objectStore,
ILogger<RiskBundleJob> logger)
{
_builder = builder ?? throw new ArgumentNullException(nameof(builder));
_signer = signer ?? throw new ArgumentNullException(nameof(signer));
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RiskBundleJobOutcome> ExecuteAsync(RiskBundleJobRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
cancellationToken.ThrowIfCancellationRequested();
var build = _builder.Build(request.BuildRequest, cancellationToken);
_logger.LogInformation("Risk bundle built with {ProviderCount} providers.", build.Manifest.Providers.Count);
var signature = await _signer.SignAsync(build.ManifestJson, cancellationToken).ConfigureAwait(false);
var signatureJson = System.Text.Json.JsonSerializer.Serialize(signature, new System.Text.Json.JsonSerializerOptions
{
PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
WriteIndented = false
});
var manifestKey = Combine(request.StoragePrefix, request.BuildRequest.ManifestFileName);
var manifestSigKey = Combine(request.StoragePrefix, request.BuildRequest.ManifestDsseFileName);
var bundleKey = Combine(request.StoragePrefix, request.BundleFileName);
var manifestStorage = await _objectStore.StoreAsync(
new RiskBundleObjectStoreOptions(manifestKey, "application/json"),
new MemoryStream(System.Text.Encoding.UTF8.GetBytes(build.ManifestJson)),
cancellationToken).ConfigureAwait(false);
var signatureStorage = await _objectStore.StoreAsync(
new RiskBundleObjectStoreOptions(manifestSigKey, "application/json"),
new MemoryStream(System.Text.Encoding.UTF8.GetBytes(signatureJson)),
cancellationToken).ConfigureAwait(false);
var bundleStorage = await _objectStore.StoreAsync(
new RiskBundleObjectStoreOptions(bundleKey, "application/gzip"),
build.BundleStream,
cancellationToken).ConfigureAwait(false);
return new RiskBundleJobOutcome(
build.Manifest,
manifestStorage,
signatureStorage,
bundleStorage,
build.ManifestJson,
signatureJson,
build.RootHash);
}
private static string Combine(string prefix, string fileName)
{
if (string.IsNullOrWhiteSpace(prefix))
{
return fileName;
}
var sanitizedPrefix = prefix.Trim('/');
return string.IsNullOrWhiteSpace(sanitizedPrefix)
? fileName
: $"{sanitizedPrefix}/{fileName}";
}
}

View File

@@ -0,0 +1,43 @@
using System.Text.Json.Serialization;
namespace StellaOps.ExportCenter.RiskBundles;
public sealed record RiskBundleProviderInput(
string ProviderId,
string SourcePath,
string Source,
bool Optional = false,
DateOnly? SnapshotDate = null);
public sealed record RiskBundleProviderEntry(
string ProviderId,
string Source,
DateOnly? SnapshotDate,
string Sha256,
long SizeBytes,
bool Optional,
string BundlePath,
string SourceFilePath,
string? SignaturePath);
public sealed record RiskBundleManifest(
string Version,
Guid BundleId,
DateTimeOffset CreatedAt,
IReadOnlyList<RiskBundleProviderEntry> Providers,
string InputsHash);
public sealed record RiskBundleBuildRequest(
Guid BundleId,
IReadOnlyList<RiskBundleProviderInput> Providers,
string? BundleFileName = null,
string BundlePrefix = "risk-bundles",
string ManifestFileName = "provider-manifest.json",
string ManifestDsseFileName = "provider-manifest.dsse",
bool AllowMissingOptional = true);
public sealed record RiskBundleBuildResult(
RiskBundleManifest Manifest,
string ManifestJson,
string RootHash,
MemoryStream BundleStream);

View File

@@ -0,0 +1,13 @@
namespace StellaOps.ExportCenter.RiskBundles;
public sealed record RiskBundleObjectStoreOptions(string StorageKey, string ContentType);
public sealed record RiskBundleStorageMetadata(string StorageKey, long SizeBytes, string ContentType);
public interface IRiskBundleObjectStore
{
Task<RiskBundleStorageMetadata> StoreAsync(
RiskBundleObjectStoreOptions options,
Stream content,
CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,84 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json.Serialization;
namespace StellaOps.ExportCenter.RiskBundles;
public interface IRiskBundleManifestSigner
{
Task<RiskBundleManifestSignatureDocument> SignAsync(string manifestJson, CancellationToken cancellationToken = default);
}
public sealed record RiskBundleManifestSignatureDocument(
[property: JsonPropertyName("payloadType")] string PayloadType,
[property: JsonPropertyName("payload")] string Payload,
[property: JsonPropertyName("signatures")] IReadOnlyList<RiskBundleManifestDsseSignature> Signatures);
public sealed record RiskBundleManifestDsseSignature(
[property: JsonPropertyName("sig")] string Signature,
[property: JsonPropertyName("keyid")] string KeyId);
public sealed class HmacRiskBundleManifestSigner : IRiskBundleManifestSigner
{
private const string DefaultPayloadType = "application/stellaops.risk-bundle.provider-manifest+json";
private readonly byte[] _key;
private readonly string _keyId;
public HmacRiskBundleManifestSigner(string key, string keyId)
{
if (string.IsNullOrWhiteSpace(key))
{
throw new ArgumentException("Signing key cannot be empty.", nameof(key));
}
_key = Encoding.UTF8.GetBytes(key);
_keyId = string.IsNullOrWhiteSpace(keyId) ? "risk-bundle-hmac" : keyId;
}
public Task<RiskBundleManifestSignatureDocument> SignAsync(string manifestJson, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(manifestJson);
cancellationToken.ThrowIfCancellationRequested();
var pae = CreatePreAuthenticationEncoding(DefaultPayloadType, manifestJson);
var signature = ComputeHmac(pae, _key);
var document = new RiskBundleManifestSignatureDocument(
DefaultPayloadType,
Convert.ToBase64String(Encoding.UTF8.GetBytes(manifestJson)),
new[] { new RiskBundleManifestDsseSignature(signature, _keyId) });
return Task.FromResult(document);
}
private static string ComputeHmac(byte[] pae, byte[] key)
{
using var hmac = new HMACSHA256(key);
var signature = hmac.ComputeHash(pae);
return Convert.ToBase64String(signature);
}
private static byte[] CreatePreAuthenticationEncoding(string payloadType, string payload)
{
// DSSE PAE: length + type + length + payload
var typeBytes = Encoding.UTF8.GetBytes(payloadType);
var payloadBytes = Encoding.UTF8.GetBytes(payload);
var builder = new List<byte>(typeBytes.Length + payloadBytes.Length + 32);
AppendLength(builder, typeBytes.Length);
builder.AddRange(typeBytes);
AppendLength(builder, payloadBytes.Length);
builder.AddRange(payloadBytes);
return builder.ToArray();
}
private static void AppendLength(List<byte> buffer, int length)
{
var text = length.ToString(CultureInfo.InvariantCulture);
var bytes = Encoding.UTF8.GetBytes(text);
buffer.AddRange(bytes);
buffer.Add(0x20); // space
}
}

View File

@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.ExportCenter/StellaOps.ExportCenter.Core/StellaOps.ExportCenter.Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
</Project>