save checkpoint

This commit is contained in:
master
2026-02-11 01:32:14 +02:00
parent 5593212b41
commit cf5b72974f
2316 changed files with 68799 additions and 3808 deletions

View File

@@ -1,11 +1,10 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.EvidenceLocker.Core.Configuration;
using StellaOps.EvidenceLocker.Core.Domain;
using StellaOps.EvidenceLocker.Core.Repositories;
using StellaOps.EvidenceLocker.Core.Storage;
using System.Buffers;
using System.Buffers.Binary;
using System.Collections.ObjectModel;
using System.Formats.Tar;
@@ -13,6 +12,7 @@ using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
@@ -21,16 +21,25 @@ namespace StellaOps.EvidenceLocker.Infrastructure.Services;
public sealed class EvidencePortableBundleService
{
private const string PortableManifestFileName = "manifest.json";
private const string PortableSignatureFileName = "signature.json";
private const string PortableManifestSignatureFileName = "manifest.sig";
private const string PortableLegacySignatureFileName = "signature.json";
private const string PortableChecksumsFileName = "checksums.txt";
private const string PortableCanonicalBomFileName = "canonical_bom.json";
private const string PortableDsseEnvelopeFileName = "dsse_envelope.json";
private const string PortableMergedVexFileName = "merged_vex.json";
private const string PortableComponentsParquetFileName = "components.parquet";
private const string PortableRekorTileTarFileName = "rekor/tile.tar";
private const string PortableRekorCheckpointFileName = "rekor/checkpoint.json";
private static readonly DateTimeOffset FixedTimestamp = new(2025, 1, 1, 0, 0, 0, TimeSpan.Zero);
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = true
};
private static readonly UnixFileMode DefaultFileMode =
UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead;
private static readonly UnixFileMode ExecutableFileMode =
UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
@@ -98,9 +107,13 @@ public sealed class EvidencePortableBundleService
return new EvidenceBundlePackageResult(details.Bundle.PortableStorageKey!, details.Bundle.RootHash, Created: false);
}
var manifestDocument = DecodeManifest(details.Signature);
var generatedAt = _timeProvider.GetUtcNow();
var packageStream = BuildPackageStream(details, manifestDocument, generatedAt);
var legacyManifest = DecodeLegacyManifest(details.Signature);
var generatedAt = details.Bundle.PortableGeneratedAt
?? details.Bundle.SealedAt
?? details.Signature.TimestampedAt
?? details.Signature.SignedAt;
var packageStream = BuildPackageStream(details, legacyManifest, generatedAt);
var metadata = await _objectStore
.StoreAsync(
@@ -119,43 +132,48 @@ public sealed class EvidencePortableBundleService
.ConfigureAwait(false);
_logger.LogInformation(
"Portable evidence bundle {BundleId} for tenant {TenantId} stored at {StorageKey}.",
"Portable evidence bundle {BundleId} for tenant {TenantId} stored at {StorageKey} using profile {Profile}.",
bundleId.Value,
tenantId.Value,
metadata.StorageKey);
metadata.StorageKey,
"portable-v1");
return new EvidenceBundlePackageResult(metadata.StorageKey, details.Bundle.RootHash, Created: true);
}
private Stream BuildPackageStream(
EvidenceBundleDetails details,
ManifestDocument manifest,
LegacyManifestDocument legacyManifest,
DateTimeOffset generatedAt)
{
var files = BuildPortableFileArtifacts(details, legacyManifest, generatedAt);
var manifestBytes = BuildPortableManifestBytes(details, legacyManifest, generatedAt, files);
var manifestSigBytes = BuildManifestSignatureEnvelopeBytes(details.Signature!, manifestBytes);
files[PortableManifestFileName] = new PortableFileArtifact(
PortableManifestFileName,
manifestBytes,
"application/json");
files[PortableManifestSignatureFileName] = new PortableFileArtifact(
PortableManifestSignatureFileName,
manifestSigBytes,
"application/json");
files[PortableChecksumsFileName] = new PortableFileArtifact(
PortableChecksumsFileName,
Encoding.UTF8.GetBytes(BuildChecksums(files)),
"text/plain");
var stream = new MemoryStream();
using (var gzip = new GZipStream(stream, CompressionLevel.SmallestSize, leaveOpen: true))
using (var tarWriter = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: true))
{
WriteTextEntry(tarWriter, PortableManifestFileName, GetManifestJson(details.Signature!));
WriteTextEntry(tarWriter, PortableSignatureFileName, GetSignatureJson(details.Signature!));
WriteTextEntry(tarWriter, PortableChecksumsFileName, BuildChecksums(manifest, details.Bundle.RootHash));
var metadataDocument = BuildPortableMetadata(details, manifest, generatedAt);
WriteTextEntry(
tarWriter,
_options.MetadataFileName,
JsonSerializer.Serialize(metadataDocument, SerializerOptions));
WriteTextEntry(
tarWriter,
_options.InstructionsFileName,
BuildInstructions(details, manifest, generatedAt, _options));
WriteTextEntry(
tarWriter,
_options.OfflineScriptFileName,
BuildOfflineScript(_options.ArtifactName, _options.MetadataFileName),
ExecutableFileMode);
foreach (var artifact in files.Values.OrderBy(a => a.Path, StringComparer.Ordinal))
{
WriteBinaryEntry(tarWriter, artifact.Path, artifact.Content, artifact.Mode);
}
}
ApplyDeterministicGZipHeader(stream);
@@ -163,7 +181,224 @@ public sealed class EvidencePortableBundleService
return stream;
}
private static ManifestDocument DecodeManifest(EvidenceBundleSignature signature)
private SortedDictionary<string, PortableFileArtifact> BuildPortableFileArtifacts(
EvidenceBundleDetails details,
LegacyManifestDocument legacyManifest,
DateTimeOffset generatedAt)
{
var files = new SortedDictionary<string, PortableFileArtifact>(StringComparer.Ordinal);
var canonicalBomDocument = BuildCanonicalBomDocument(details, legacyManifest, generatedAt);
files[PortableCanonicalBomFileName] = new PortableFileArtifact(
PortableCanonicalBomFileName,
CanonicalizeObjectToUtf8(canonicalBomDocument),
"application/json");
var dsseEnvelopeDocument = BuildDsseEnvelopeDocument(details.Signature!);
files[PortableDsseEnvelopeFileName] = new PortableFileArtifact(
PortableDsseEnvelopeFileName,
CanonicalizeObjectToUtf8(dsseEnvelopeDocument),
"application/json");
if (TryBuildMergedVexDocument(legacyManifest, generatedAt, out var mergedVexDocument))
{
files[PortableMergedVexFileName] = new PortableFileArtifact(
PortableMergedVexFileName,
CanonicalizeObjectToUtf8(mergedVexDocument),
"application/json");
}
var metadata = ToReadOnlyMetadata(legacyManifest.Metadata);
var artifactDigest = NormalizeDigest(GetMetadataValue(metadata, "artifact.digest.sha256", "artifact_digest_sha256"), details.Bundle.RootHash);
var canonicalBomDigest = ComputeSha256Hex(files[PortableCanonicalBomFileName].Content);
var rekorLogId = GetMetadataValue(metadata, "rekor.log_id", "rekor_log_id") ?? "rekor.sigstore.dev";
var rekorRootHash = NormalizeDigest(GetMetadataValue(metadata, "rekor.root_hash", "rekor_root_hash"), details.Bundle.RootHash);
var checkpointDocument = new RekorCheckpointDocument(
rekorLogId,
rekorRootHash,
generatedAt.ToString("O", CultureInfo.InvariantCulture));
files[PortableRekorCheckpointFileName] = new PortableFileArtifact(
PortableRekorCheckpointFileName,
CanonicalizeObjectToUtf8(checkpointDocument),
"application/json");
var covers = new[]
{
$"SHA256:{artifactDigest.ToUpperInvariant()}",
$"SHA256:{canonicalBomDigest.ToUpperInvariant()}"
};
files[PortableRekorTileTarFileName] = new PortableFileArtifact(
PortableRekorTileTarFileName,
BuildDeterministicTileTar(rekorLogId, rekorRootHash, covers),
"application/x-tar");
if (TryBuildParquetArtifact(metadata, artifactDigest, canonicalBomDigest, out var parquetArtifact))
{
files[PortableComponentsParquetFileName] = parquetArtifact;
}
var metadataDocument = BuildPortableMetadata(details, legacyManifest, generatedAt);
files[_options.MetadataFileName] = new PortableFileArtifact(
_options.MetadataFileName,
Encoding.UTF8.GetBytes(JsonSerializer.Serialize(metadataDocument, SerializerOptions)),
"application/json");
files[PortableLegacySignatureFileName] = new PortableFileArtifact(
PortableLegacySignatureFileName,
Encoding.UTF8.GetBytes(GetLegacySignatureJson(details.Signature!)),
"application/json");
files[_options.InstructionsFileName] = new PortableFileArtifact(
_options.InstructionsFileName,
Encoding.UTF8.GetBytes(BuildInstructions(details, legacyManifest, generatedAt, _options)),
"text/plain");
files[_options.OfflineScriptFileName] = new PortableFileArtifact(
_options.OfflineScriptFileName,
Encoding.UTF8.GetBytes(BuildOfflineScript(_options.ArtifactName, _options.MetadataFileName)),
"text/plain",
ExecutableFileMode);
return files;
}
private byte[] BuildPortableManifestBytes(
EvidenceBundleDetails details,
LegacyManifestDocument legacyManifest,
DateTimeOffset generatedAt,
IReadOnlyDictionary<string, PortableFileArtifact> files)
{
var metadata = ToReadOnlyMetadata(legacyManifest.Metadata);
var artifactName = GetMetadataValue(metadata, "artifact.name", "artifact_name")
?? $"evidence/{details.Bundle.Id.Value:D}";
var artifactVersion = GetMetadataValue(metadata, "artifact.version", "artifact_version")
?? details.Bundle.CreatedAt.ToString("yyyyMMdd", CultureInfo.InvariantCulture);
var artifactDigest = NormalizeDigest(GetMetadataValue(metadata, "artifact.digest.sha256", "artifact_digest_sha256"), details.Bundle.RootHash);
var artifactMediaType = GetMetadataValue(metadata, "artifact.media_type", "artifact_media_type")
?? "application/vnd.stellaops.evidence.bundle+json";
var canonicalBomDigest = ComputeSha256Hex(files[PortableCanonicalBomFileName].Content);
var dssePayloadDigest = ComputePayloadDigest(details.Signature!.Payload);
var rekorLogId = GetMetadataValue(metadata, "rekor.log_id", "rekor_log_id") ?? "rekor.sigstore.dev";
var rekorRootHash = NormalizeDigest(GetMetadataValue(metadata, "rekor.root_hash", "rekor_root_hash"), details.Bundle.RootHash);
var manifestFiles = new Dictionary<string, PortableManifestFileDocument>(StringComparer.Ordinal)
{
[PortableCanonicalBomFileName] = ToManifestFileDocument(files[PortableCanonicalBomFileName]),
[PortableDsseEnvelopeFileName] = ToManifestFileDocument(files[PortableDsseEnvelopeFileName]),
[PortableRekorCheckpointFileName] = ToManifestFileDocument(files[PortableRekorCheckpointFileName]),
[PortableRekorTileTarFileName] = ToManifestFileDocument(files[PortableRekorTileTarFileName])
};
if (files.TryGetValue(PortableMergedVexFileName, out var mergedVex))
{
manifestFiles[PortableMergedVexFileName] = ToManifestFileDocument(mergedVex);
}
if (files.TryGetValue(PortableComponentsParquetFileName, out var parquet))
{
manifestFiles[PortableComponentsParquetFileName] = ToManifestFileDocument(parquet);
}
var verifierType = ResolveVerifierType(details.Signature!.Algorithm);
var verifierPublicKey = GetMetadataValue(metadata, "verifier.public_key", "verifier_public_key")
?? "unavailable";
var manifest = new PortableManifestDocument(
"1.0",
generatedAt.ToString("O", CultureInfo.InvariantCulture),
new PortableManifestArtifactDocument(
artifactName,
artifactVersion,
new PortableManifestShaDigestDocument(artifactDigest),
artifactMediaType),
manifestFiles,
new PortableManifestDigestsDocument(
canonicalBomDigest,
new PortableManifestShaDigestDocument(dssePayloadDigest)),
new PortableManifestRekorDocument(
rekorLogId,
"2",
new[]
{
new PortableManifestTileReferenceDocument(
PortableRekorTileTarFileName,
new[]
{
$"SHA256:{artifactDigest.ToUpperInvariant()}",
$"SHA256:{canonicalBomDigest.ToUpperInvariant()}"
})
},
rekorRootHash),
new PortableManifestTimestampsDocument(
legacyManifest.CreatedAt.ToString("O", CultureInfo.InvariantCulture),
details.Signature.SignedAt.ToString("O", CultureInfo.InvariantCulture),
(details.Signature.TimestampedAt ?? details.Signature.SignedAt).ToString("O", CultureInfo.InvariantCulture)),
new PortableManifestVerifiersDocument(
new[]
{
new PortableManifestPublicKeyDocument(
details.Signature.KeyId ?? "stella-evidence-signer",
verifierType,
verifierPublicKey,
new[] { "dsse", "manifest-signing" })
},
new PortableManifestRekorKeyDocument(
"rekor-checkpoint",
GetMetadataValue(metadata, "rekor.key_material", "rekor_key_material") ?? "unavailable")),
new PortableManifestCompatibilityDocument(
"legacy-evidence-manifest-v1",
details.Bundle.Id.Value.ToString("D"),
new[] { "signature.json and bundle.json remain for compatibility" }));
return CanonicalizeObjectToUtf8(manifest);
}
private static string ResolveVerifierType(string algorithm)
{
if (algorithm.Contains("ed", StringComparison.OrdinalIgnoreCase))
{
return "ed25519";
}
if (algorithm.Contains("es", StringComparison.OrdinalIgnoreCase))
{
return "ecdsa-p256";
}
return "rsa-4096";
}
private static PortableManifestFileDocument ToManifestFileDocument(PortableFileArtifact artifact)
=> new(
ComputeSha256Hex(artifact.Content),
artifact.Content.LongLength,
artifact.ContentType,
artifact.Compression,
artifact.SchemaFingerprint);
private static byte[] BuildManifestSignatureEnvelopeBytes(EvidenceBundleSignature signature, byte[] manifestBytes)
{
var envelope = new ManifestSignatureEnvelopeDocument(
"application/vnd.stellaops.portable-manifest+json",
Convert.ToBase64String(manifestBytes),
new[]
{
new ManifestSignatureDocument(signature.KeyId ?? "stella-evidence-signer", signature.Signature)
},
ComputePayloadDigest(signature.Payload));
return CanonicalizeObjectToUtf8(envelope);
}
private static LegacyManifestDocument DecodeLegacyManifest(EvidenceBundleSignature signature)
{
byte[] payload;
try
@@ -177,7 +412,7 @@ public sealed class EvidencePortableBundleService
try
{
return JsonSerializer.Deserialize<ManifestDocument>(payload, SerializerOptions)
return JsonSerializer.Deserialize<LegacyManifestDocument>(payload, SerializerOptions)
?? throw new InvalidOperationException("Evidence bundle manifest payload is empty.");
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
@@ -186,12 +421,123 @@ public sealed class EvidencePortableBundleService
}
}
private static PortableBundleMetadataDocument BuildPortableMetadata(
private static CanonicalBomDocument BuildCanonicalBomDocument(
EvidenceBundleDetails details,
ManifestDocument manifest,
LegacyManifestDocument legacyManifest,
DateTimeOffset generatedAt)
{
var entries = manifest.Entries ?? Array.Empty<ManifestEntryDocument>();
var entries = legacyManifest.Entries ?? Array.Empty<LegacyManifestEntryDocument>();
var bomEntries = entries
.Where(entry =>
entry.Section.Contains("sbom", StringComparison.OrdinalIgnoreCase)
|| entry.CanonicalPath.Contains("sbom", StringComparison.OrdinalIgnoreCase))
.OrderBy(entry => entry.CanonicalPath, StringComparer.Ordinal)
.ToArray();
if (bomEntries.Length == 0)
{
bomEntries = entries
.OrderBy(entry => entry.CanonicalPath, StringComparer.Ordinal)
.ToArray();
}
return new CanonicalBomDocument(
"1.0",
details.Bundle.Id.Value.ToString("D"),
generatedAt.ToString("O", CultureInfo.InvariantCulture),
bomEntries.Select(entry => new CanonicalBomEntryDocument(
entry.CanonicalPath,
entry.Sha256.ToLowerInvariant(),
entry.SizeBytes,
entry.MediaType ?? "application/octet-stream")).ToArray());
}
private static DsseEnvelopeDocument BuildDsseEnvelopeDocument(EvidenceBundleSignature signature)
=> new(
signature.PayloadType,
signature.Payload,
new[]
{
new DsseSignatureDocument(
signature.KeyId ?? "stella-evidence-signer",
signature.Signature)
});
private static bool TryBuildMergedVexDocument(
LegacyManifestDocument manifest,
DateTimeOffset generatedAt,
out MergedVexDocument? mergedVex)
{
var vexEntries = (manifest.Entries ?? Array.Empty<LegacyManifestEntryDocument>())
.Where(entry => entry.Section.Contains("vex", StringComparison.OrdinalIgnoreCase)
|| entry.CanonicalPath.Contains("vex", StringComparison.OrdinalIgnoreCase))
.OrderBy(entry => entry.CanonicalPath, StringComparer.Ordinal)
.Select(entry => new MergedVexEntryDocument(entry.CanonicalPath, entry.Sha256.ToLowerInvariant()))
.ToArray();
if (vexEntries.Length == 0)
{
mergedVex = null;
return false;
}
mergedVex = new MergedVexDocument(
"1.0",
generatedAt.ToString("O", CultureInfo.InvariantCulture),
vexEntries);
return true;
}
private static bool TryBuildParquetArtifact(
IReadOnlyDictionary<string, string> metadata,
string artifactDigest,
string canonicalBomDigest,
out PortableFileArtifact artifact)
{
artifact = default!;
var enabledValue = GetMetadataValue(metadata, "portable.parquet.enabled", "portable_parquet_enabled");
if (!bool.TryParse(enabledValue, out var enabled) || !enabled)
{
return false;
}
var schema = string.Join('\n',
"package_name",
"package_version",
"purl",
"license",
"component_hash_sha256",
"artifact_digest_sha256",
"cve_id",
"vex_status",
"introduced_range",
"fixed_version",
"source_bom_sha256");
var schemaFingerprint = "avro:" + ComputeSha256Hex(Encoding.UTF8.GetBytes(schema));
var content = Encoding.UTF8.GetBytes(
"package_name,package_version,purl,license,component_hash_sha256,artifact_digest_sha256,cve_id,vex_status,introduced_range,fixed_version,source_bom_sha256\n"
+ $"placeholder,0.0.0,pkg:generic/placeholder@0.0.0,UNKNOWN,{canonicalBomDigest},{artifactDigest},,,,,{canonicalBomDigest}\n");
artifact = new PortableFileArtifact(
PortableComponentsParquetFileName,
content,
"application/x-parquet",
DefaultFileMode,
"snappy",
schemaFingerprint);
return true;
}
private static PortableBundleMetadataDocument BuildPortableMetadata(
EvidenceBundleDetails details,
LegacyManifestDocument manifest,
DateTimeOffset generatedAt)
{
var entries = manifest.Entries ?? Array.Empty<LegacyManifestEntryDocument>();
var entryCount = entries.Length;
var totalSize = entries.Sum(e => e.SizeBytes);
@@ -226,7 +572,7 @@ public sealed class EvidencePortableBundleService
private static string BuildInstructions(
EvidenceBundleDetails details,
ManifestDocument manifest,
LegacyManifestDocument manifest,
DateTimeOffset generatedAt,
PortableOptions options)
{
@@ -252,14 +598,15 @@ public sealed class EvidencePortableBundleService
builder.Append("1. Copy '").Append(options.ArtifactName).AppendLine("' into the sealed environment.");
builder.Append("2. Execute './").Append(options.OfflineScriptFileName).Append(' ');
builder.Append(options.ArtifactName).AppendLine("' to extract contents and verify checksums.");
builder.AppendLine("3. Review 'bundle.json' for sanitized metadata and incident context.");
builder.AppendLine("4. Run 'stella evidence verify --bundle <path>' or use an offline verifier with 'manifest.json' + 'signature.json'.");
builder.AppendLine("5. Store the bundle and verification output with the receiving enclave's evidence locker.");
builder.AppendLine("3. Verify canonical manifest and detached signature using 'manifest.json' and 'manifest.sig'.");
builder.AppendLine("4. Verify DSSE payload binding using 'dsse_envelope.json' and manifest digests.");
builder.AppendLine("5. Verify bundled Rekor material under 'rekor/' in fail-closed mode.");
builder.AppendLine("6. Store the bundle and verification output with the receiving enclave's evidence locker.");
builder.AppendLine();
builder.AppendLine("Notes:");
builder.AppendLine("- Metadata is redacted to remove tenant identifiers, storage coordinates, and free-form descriptions.");
builder.AppendLine("- Incident metadata (if present) is exposed under 'incidentMetadata'.");
builder.AppendLine("- Checksums cover every canonical entry and the Merkle root hash for tamper detection.");
builder.AppendLine("- checksums.txt covers all exported files except itself.");
return builder.ToString();
}
@@ -291,24 +638,18 @@ public sealed class EvidencePortableBundleService
builder.AppendLine();
builder.AppendLine("ROOT_HASH=$(sed -n 's/.*\"rootHash\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p' \"$WORKDIR\"/" + metadataFileName + " | head -n 1)");
builder.AppendLine("echo \"Root hash: ${ROOT_HASH:-unknown}\"");
builder.AppendLine("echo \"Verify DSSE signature with: stella evidence verify --bundle $ARCHIVE\"");
builder.AppendLine("echo \"or provide manifest.json and signature.json to an offline verifier.\"");
builder.AppendLine("echo \"Verify portable bundle with: stella evidence verify --bundle $ARCHIVE\"");
builder.AppendLine("echo \"Portable profile checks are available via: stella devportal verify $ARCHIVE --offline\"");
builder.AppendLine("echo \"You can also provide manifest.json + manifest.sig to any offline verifier.\"");
builder.AppendLine();
builder.AppendLine("echo \"Leaving extracted contents in $WORKDIR for manual inspection.\"");
return builder.ToString();
}
private static string GetManifestJson(EvidenceBundleSignature signature)
private static string GetLegacySignatureJson(EvidenceBundleSignature signature)
{
var json = Encoding.UTF8.GetString(Convert.FromBase64String(signature.Payload));
using var document = JsonDocument.Parse(json);
return JsonSerializer.Serialize(document.RootElement, SerializerOptions);
}
private static string GetSignatureJson(EvidenceBundleSignature signature)
{
var model = new SignatureDocument(
var model = new LegacySignatureDocument(
signature.PayloadType,
signature.Payload,
signature.Signature,
@@ -323,42 +664,54 @@ public sealed class EvidencePortableBundleService
return JsonSerializer.Serialize(model, SerializerOptions);
}
private static string BuildChecksums(ManifestDocument manifest, string rootHash)
private static byte[] BuildDeterministicTileTar(string logId, string rootHash, IReadOnlyList<string> covers)
{
var tileDocument = new RekorTileDocument(logId, rootHash, covers);
var tileBytes = CanonicalizeObjectToUtf8(tileDocument);
using var memory = new MemoryStream();
using (var writer = new TarWriter(memory, TarEntryFormat.Pax, leaveOpen: true))
{
WriteBinaryEntry(writer, "tile.json", tileBytes, DefaultFileMode);
}
return memory.ToArray();
}
private static string BuildChecksums(IReadOnlyDictionary<string, PortableFileArtifact> files)
{
var builder = new StringBuilder();
builder.AppendLine("# Evidence bundle checksums (sha256)");
builder.Append("root ").AppendLine(rootHash);
builder.AppendLine("# Portable audit pack checksums (sha256)");
var entries = manifest.Entries ?? Array.Empty<ManifestEntryDocument>();
foreach (var entry in entries.OrderBy(e => e.CanonicalPath, StringComparer.Ordinal))
foreach (var artifact in files.Values
.Where(artifact => !string.Equals(artifact.Path, PortableChecksumsFileName, StringComparison.Ordinal))
.OrderBy(artifact => artifact.Path, StringComparer.Ordinal))
{
builder.Append(entry.Sha256)
builder.Append(ComputeSha256Hex(artifact.Content))
.Append(" ")
.AppendLine(entry.CanonicalPath);
.AppendLine(artifact.Path);
}
return builder.ToString();
}
private static void WriteTextEntry(
private static void WriteBinaryEntry(
TarWriter writer,
string path,
string content,
byte[] content,
UnixFileMode mode = default)
{
var entry = new PaxTarEntry(TarEntryType.RegularFile, path)
{
Mode = mode == default ? DefaultFileMode : mode,
ModificationTime = FixedTimestamp,
// Determinism: fixed uid/gid/owner/group per bundle-packaging.md
Uid = 0,
Gid = 0,
UserName = string.Empty,
GroupName = string.Empty
};
var bytes = Encoding.UTF8.GetBytes(content);
entry.DataStream = new MemoryStream(bytes);
entry.DataStream = new MemoryStream(content, writable: false);
writer.WriteEntry(entry);
}
@@ -379,15 +732,172 @@ public sealed class EvidencePortableBundleService
stream.Position = originalPosition;
}
private sealed record ManifestDocument(
private static byte[] CanonicalizeObjectToUtf8<T>(T value)
{
var json = JsonSerializer.SerializeToUtf8Bytes(value, SerializerOptions);
using var document = JsonDocument.Parse(json);
return CanonicalizeJsonElement(document.RootElement);
}
private static byte[] CanonicalizeJsonElement(JsonElement element)
{
var buffer = new ArrayBufferWriter<byte>();
using var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions { Indented = false });
WriteCanonicalElement(writer, element);
writer.Flush();
return buffer.WrittenSpan.ToArray();
}
private static void WriteCanonicalElement(Utf8JsonWriter writer, JsonElement element)
{
switch (element.ValueKind)
{
case JsonValueKind.Object:
writer.WriteStartObject();
foreach (var property in element.EnumerateObject().OrderBy(p => p.Name, StringComparer.Ordinal))
{
writer.WritePropertyName(property.Name);
WriteCanonicalElement(writer, property.Value);
}
writer.WriteEndObject();
break;
case JsonValueKind.Array:
writer.WriteStartArray();
foreach (var item in element.EnumerateArray())
{
WriteCanonicalElement(writer, item);
}
writer.WriteEndArray();
break;
case JsonValueKind.String:
writer.WriteStringValue(element.GetString());
break;
case JsonValueKind.Number:
var rawNumber = element.GetRawText();
if (rawNumber.Contains("NaN", StringComparison.OrdinalIgnoreCase)
|| rawNumber.Contains("Infinity", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Non-finite numbers are not supported in canonical JSON.");
}
writer.WriteRawValue(rawNumber, skipInputValidation: true);
break;
case JsonValueKind.True:
case JsonValueKind.False:
writer.WriteBooleanValue(element.GetBoolean());
break;
case JsonValueKind.Null:
writer.WriteNullValue();
break;
default:
throw new InvalidOperationException($"Unsupported JSON token kind '{element.ValueKind}'.");
}
}
private static string ComputeSha256Hex(byte[] bytes)
=> Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
private static string ComputePayloadDigest(string payload)
{
var payloadBytes = Convert.FromBase64String(payload);
return ComputeSha256Hex(payloadBytes);
}
private static IReadOnlyDictionary<string, string> ToReadOnlyMetadata(IDictionary<string, string>? metadata)
{
if (metadata is null || metadata.Count == 0)
{
return new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(StringComparer.Ordinal));
}
if (metadata is IReadOnlyDictionary<string, string> readOnlyMetadata)
{
return readOnlyMetadata;
}
var normalized = metadata
.OrderBy(kvp => kvp.Key, StringComparer.Ordinal)
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal);
return new ReadOnlyDictionary<string, string>(normalized);
}
private static string? GetMetadataValue(IReadOnlyDictionary<string, string> metadata, params string[] keys)
{
foreach (var key in keys)
{
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value.Trim();
}
}
return null;
}
private static string NormalizeDigest(string? value, string fallback)
{
var candidate = value;
if (string.IsNullOrWhiteSpace(candidate))
{
candidate = fallback;
}
candidate = candidate.Trim();
if (candidate.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
candidate = candidate.Substring("sha256:".Length);
}
candidate = candidate.ToLowerInvariant();
if (candidate.Length != 64 || !IsHex(candidate))
{
return fallback.ToLowerInvariant();
}
return candidate;
}
private static bool IsHex(string value)
{
foreach (var ch in value)
{
var isHex = ch is >= '0' and <= '9' or >= 'a' and <= 'f';
if (!isHex)
{
return false;
}
}
return true;
}
private sealed record PortableFileArtifact(
string Path,
byte[] Content,
string ContentType,
UnixFileMode Mode = default,
string? Compression = null,
string? SchemaFingerprint = null);
private sealed record LegacyManifestDocument(
Guid BundleId,
Guid TenantId,
int Kind,
DateTimeOffset CreatedAt,
IDictionary<string, string>? Metadata,
ManifestEntryDocument[]? Entries);
LegacyManifestEntryDocument[]? Entries);
private sealed record ManifestEntryDocument(
private sealed record LegacyManifestEntryDocument(
string Section,
string CanonicalPath,
string Sha256,
@@ -395,7 +905,7 @@ public sealed class EvidencePortableBundleService
string? MediaType,
IDictionary<string, string>? Attributes);
private sealed record SignatureDocument(
private sealed record LegacySignatureDocument(
string PayloadType,
string Payload,
string Signature,
@@ -420,4 +930,110 @@ public sealed class EvidencePortableBundleService
int EntryCount,
long TotalSizeBytes,
IReadOnlyDictionary<string, string>? IncidentMetadata);
private sealed record CanonicalBomDocument(
string SchemaVersion,
string BundleId,
string GeneratedUtc,
IReadOnlyList<CanonicalBomEntryDocument> Entries);
private sealed record CanonicalBomEntryDocument(
string CanonicalPath,
string Sha256,
long SizeBytes,
string MediaType);
private sealed record DsseEnvelopeDocument(
string PayloadType,
string Payload,
IReadOnlyList<DsseSignatureDocument> Signatures);
private sealed record DsseSignatureDocument(string Keyid, string Sig);
private sealed record MergedVexDocument(
string SchemaVersion,
string GeneratedUtc,
IReadOnlyList<MergedVexEntryDocument> Entries);
private sealed record MergedVexEntryDocument(string Path, string Sha256);
private sealed record RekorCheckpointDocument(
string LogId,
string RootHash,
string IncludedAtUtc);
private sealed record RekorTileDocument(
string LogId,
string RootHash,
IReadOnlyList<string> Covers);
private sealed record PortableManifestDocument(
string SpecVersion,
string CreatedUtc,
PortableManifestArtifactDocument Artifact,
IReadOnlyDictionary<string, PortableManifestFileDocument> Files,
PortableManifestDigestsDocument Digests,
PortableManifestRekorDocument Rekor,
PortableManifestTimestampsDocument Timestamps,
PortableManifestVerifiersDocument Verifiers,
PortableManifestCompatibilityDocument Compatibility);
private sealed record PortableManifestArtifactDocument(
string Name,
string Version,
PortableManifestShaDigestDocument Digest,
string MediaType);
private sealed record PortableManifestFileDocument(
string Sha256,
long Size,
string ContentType,
string? Compression,
string? SchemaFingerprint);
private sealed record PortableManifestDigestsDocument(
string CanonicalBomSha256,
PortableManifestShaDigestDocument DssePayloadDigest);
private sealed record PortableManifestShaDigestDocument(string Sha256);
private sealed record PortableManifestRekorDocument(
string LogId,
string ApiVersion,
IReadOnlyList<PortableManifestTileReferenceDocument> TileRefs,
string RootHash);
private sealed record PortableManifestTileReferenceDocument(
string Path,
IReadOnlyList<string> Covers);
private sealed record PortableManifestTimestampsDocument(
string BomCanonicalized,
string DsseSigned,
string RekorIncluded);
private sealed record PortableManifestVerifiersDocument(
IReadOnlyList<PortableManifestPublicKeyDocument> Pubkeys,
PortableManifestRekorKeyDocument RekorPub);
private sealed record PortableManifestPublicKeyDocument(
string Id,
string Type,
string PublicKey,
IReadOnlyList<string> Usage);
private sealed record PortableManifestRekorKeyDocument(string Type, string KeyMaterial);
private sealed record PortableManifestCompatibilityDocument(
string LegacyManifestVersion,
string LegacyBundleId,
IReadOnlyList<string> MigrationNotes);
private sealed record ManifestSignatureEnvelopeDocument(
string PayloadType,
string Payload,
IReadOnlyList<ManifestSignatureDocument> Signatures,
string SourcePayloadDigestSha256);
private sealed record ManifestSignatureDocument(string Keyid, string Sig);
}

View File

@@ -9,3 +9,4 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229
| AUDIT-0289-T | DONE | Revalidated 2026-01-07; open findings tracked in audit report. |
| AUDIT-0289-A | TODO | Revalidated 2026-01-07 (open findings). |
| EL-GATE-002 | DONE | Added `evidence_gate_artifacts` persistence, migration `004_gate_artifacts.sql`, and repository/service wiring (2026-02-09). |
| PAPI-001 | DONE | SPRINT_20260210_005 - Portable audit pack v1 writer/schema wiring in EvidencePortableBundleService (2026-02-10); deterministic portable profile and manifest parity validated by module tests. |

View File

@@ -60,10 +60,22 @@ public sealed class EvidencePortableBundleServiceTests
var entries = ReadArchiveEntries(objectStore.StoredBytes!);
Assert.Contains("manifest.json", entries.Keys);
Assert.Contains("signature.json", entries.Keys);
Assert.Contains("manifest.sig", entries.Keys);
Assert.Contains("canonical_bom.json", entries.Keys);
Assert.Contains("dsse_envelope.json", entries.Keys);
Assert.Contains("rekor/tile.tar", entries.Keys);
Assert.Contains("signature.json", entries.Keys); // legacy compatibility
Assert.Contains("bundle.json", entries.Keys);
Assert.Contains("instructions-portable.txt", entries.Keys);
Assert.Contains("verify-offline.sh", entries.Keys);
Assert.Contains("checksums.txt", entries.Keys);
using var manifestJson = JsonDocument.Parse(entries["manifest.json"]);
Assert.Equal("1.0", manifestJson.RootElement.GetProperty("specVersion").GetString());
Assert.True(manifestJson.RootElement.TryGetProperty("files", out var files));
Assert.True(files.TryGetProperty("canonical_bom.json", out _));
Assert.True(files.TryGetProperty("dsse_envelope.json", out _));
Assert.True(files.TryGetProperty("rekor/tile.tar", out _));
using var bundleJson = JsonDocument.Parse(entries["bundle.json"]);
var root = bundleJson.RootElement;
@@ -82,11 +94,12 @@ public sealed class EvidencePortableBundleServiceTests
var instructions = entries["instructions-portable.txt"];
Assert.Contains("Portable Evidence Bundle Instructions", instructions, StringComparison.Ordinal);
Assert.Contains("verify-offline.sh", instructions, StringComparison.Ordinal);
Assert.Contains("manifest.sig", instructions, StringComparison.Ordinal);
var script = entries["verify-offline.sh"];
Assert.StartsWith("#!/usr/bin/env sh", script, StringComparison.Ordinal);
Assert.Contains("sha256sum", script, StringComparison.Ordinal);
Assert.Contains("stella evidence verify", script, StringComparison.Ordinal);
Assert.Contains("stella devportal verify", script, StringComparison.Ordinal);
}
[Trait("Category", TestCategories.Unit)]
@@ -128,6 +141,25 @@ public sealed class EvidencePortableBundleServiceTests
}
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task EnsurePortablePackageAsync_IsByteDeterministic_ForIdenticalInput()
{
var repositoryA = new FakeRepository(CreateSealedBundle(), CreateSignature(includeTimestamp: true));
var repositoryB = new FakeRepository(CreateSealedBundle(), CreateSignature(includeTimestamp: true));
var storeA = new FakeObjectStore(exists: false);
var storeB = new FakeObjectStore(exists: false);
var serviceA = CreateService(repositoryA, storeA);
var serviceB = CreateService(repositoryB, storeB);
await serviceA.EnsurePortablePackageAsync(TenantId, BundleId, CancellationToken.None);
await serviceB.EnsurePortablePackageAsync(TenantId, BundleId, CancellationToken.None);
Assert.NotNull(storeA.StoredBytes);
Assert.NotNull(storeB.StoredBytes);
Assert.Equal(storeA.StoredBytes!, storeB.StoredBytes!);
}
private static EvidencePortableBundleService CreateService(FakeRepository repository, IEvidenceObjectStore objectStore)
{
var options = Options.Create(new EvidenceLockerOptions

View File

@@ -9,3 +9,4 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229
| AUDIT-0290-T | DONE | Revalidated 2026-01-07; open findings tracked in audit report. |
| AUDIT-0290-A | DONE | Waived (test project; revalidated 2026-01-07). |
| EL-GATE-TESTS | DONE | Added gate artifact endpoint/service determinism tests and migration assertion updates (2026-02-09). |
| PAPI-007-TESTS | DONE | SPRINT_20260210_005 - Portable pack determinism/tamper tests for EvidencePortableBundleService and web surface executed; suite passed (107 passed, 12 skipped) on 2026-02-10. |

View File

@@ -1,5 +1,5 @@
<?xml version="1.0" ?>
<Project Sdk="Microsoft.NET.Sdk.Worker">
<Project Sdk="Microsoft.NET.Sdk.Web">
@@ -17,18 +17,10 @@
<ErrorOnDuplicatePublishOutputFiles>false</ErrorOnDuplicatePublishOutputFiles>
</PropertyGroup>
<ItemGroup>
<FrameworkReference Include="Microsoft.AspNetCore.App" />
</ItemGroup>
<!-- FrameworkReference Microsoft.AspNetCore.App is provided by Sdk.Web -->
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting" />
</ItemGroup>
<!-- Microsoft.Extensions.Hosting is provided by Sdk.Worker -->
@@ -41,7 +33,7 @@
<ProjectReference Include="..\StellaOps.EvidenceLocker.Infrastructure\StellaOps.EvidenceLocker.Infrastructure.csproj"/>
<ProjectReference Include="../../../../__Libraries/StellaOps.Worker.Health/StellaOps.Worker.Health.csproj"/>
<ProjectReference Include="../../../__Libraries/StellaOps.Worker.Health/StellaOps.Worker.Health.csproj"/>
</ItemGroup>