This commit is contained in:
StellaOps Bot
2025-11-27 21:09:47 +02:00
parent e901d31acf
commit cfa2274d31
15 changed files with 123 additions and 47 deletions

View File

@@ -16,6 +16,8 @@ public sealed class DeterminismContext
ConcurrencyLimit = concurrencyLimit;
}
public bool IsDeterminismEnabled => FixedClock || RngSeed.HasValue || ConcurrencyLimit.HasValue || FilterLogs;
public bool FixedClock { get; }
public DateTimeOffset FixedInstantUtc { get; }

View File

@@ -74,6 +74,12 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
var payloads = CollectPayloads(context);
await PersistRubyPackagesAsync(context, cancellationToken).ConfigureAwait(false);
var determinismPayload = BuildDeterminismPayload(context, payloads, out var merkleRoot);
if (determinismPayload is not null)
{
payloads.Add(determinismPayload);
}
if (payloads.Count == 0)
{
_metrics.RecordSurfaceManifestSkipped(context);
@@ -96,7 +102,12 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
Payloads: payloads,
Component: "scanner.worker",
Version: _componentVersion,
WorkerInstance: Environment.MachineName);
WorkerInstance: Environment.MachineName,
DeterminismMerkleRoot: merkleRoot,
ReplayBundleUri: GetReplayBundleUri(context),
ReplayBundleHash: GetReplayBundleHash(context),
ReplayPolicyPin: GetPin(context, "determinism.policy"),
ReplayFeedPin: GetPin(context, "determinism.feed"));
var result = await _publisher.PublishAsync(request, cancellationToken).ConfigureAwait(false);
@@ -233,8 +244,9 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
return payloads;
}
private SurfaceManifestPayload? BuildDeterminismPayload(ScanJobContext context, IEnumerable<SurfaceManifestPayload> payloads)
private SurfaceManifestPayload? BuildDeterminismPayload(ScanJobContext context, IEnumerable<SurfaceManifestPayload> payloads, out string? merkleRoot)
{
merkleRoot = null;
var pins = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
if (context.Lease.Metadata.TryGetValue("determinism.feed", out var feed) && !string.IsNullOrWhiteSpace(feed))
{
@@ -246,12 +258,8 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
pins["policy"] = policy;
}
var artifactHashes = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var payload in payloads)
{
var digest = ComputeDigest(payload.Content.Span);
artifactHashes[payload.Kind] = digest;
}
var (artifactHashes, merkle) = ComputeDeterminismHashes(payloads);
merkleRoot = merkle;
var report = new
{
@@ -261,9 +269,13 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
filterLogs = _determinism.FilterLogs,
concurrencyLimit = _determinism.ConcurrencyLimit,
pins = pins,
artifacts = artifactHashes
artifacts = artifactHashes,
merkleRoot = merkle
};
var evidence = new Determinism.DeterminismEvidence(artifactHashes, merkle);
context.Analysis.Set(ScanAnalysisKeys.DeterminismEvidence, evidence);
var json = JsonSerializer.Serialize(report, JsonOptions);
return new SurfaceManifestPayload(
ArtifactDocumentType.SurfaceObservation,
@@ -274,6 +286,46 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
View: "replay");
}
private static (Dictionary<string, string> Hashes, string MerkleRoot) ComputeDeterminismHashes(IEnumerable<SurfaceManifestPayload> payloads)
{
var map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
using var sha = SHA256.Create();
foreach (var payload in payloads.OrderBy(p => p.Kind, StringComparer.Ordinal))
{
var digest = ComputeDigest(payload.Content.Span);
map[payload.Kind] = digest;
}
// Build Merkle-like root by hashing the ordered list of kind:digest lines.
var builder = new StringBuilder();
foreach (var kvp in map.OrderBy(kv => kv.Key, StringComparer.Ordinal))
{
builder.Append(kvp.Key).Append(':').Append(kvp.Value).Append('\n');
}
var rootBytes = Encoding.UTF8.GetBytes(builder.ToString());
var rootHash = sha.ComputeHash(rootBytes);
var merkleRoot = Convert.ToHexString(rootHash).ToLowerInvariant();
return (map, merkleRoot);
}
private static string? GetReplayBundleUri(ScanJobContext context)
=> context.Lease.Metadata.TryGetValue("replay.bundle.uri", out var value) && !string.IsNullOrWhiteSpace(value)
? value.Trim()
: null;
private static string? GetReplayBundleHash(ScanJobContext context)
=> context.Lease.Metadata.TryGetValue("replay.bundle.sha256", out var value) && !string.IsNullOrWhiteSpace(value)
? value.Trim().ToLowerInvariant()
: null;
private static string? GetPin(ScanJobContext context, string key)
=> context.Lease.Metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)
? value.Trim()
: null;
private async Task PersistRubyPackagesAsync(ScanJobContext context, CancellationToken cancellationToken)
{
if (!context.Analysis.TryGet<ReadOnlyDictionary<string, LanguageAnalyzerResult>>(ScanAnalysisKeys.LanguageAnalyzerResults, out var results))

View File

@@ -87,6 +87,8 @@ builder.Services.AddSingleton<IEntryTraceExecutionService, EntryTraceExecutionSe
builder.Services.AddSingleton<ReachabilityUnionWriter>();
builder.Services.AddSingleton<ReachabilityUnionPublisher>();
builder.Services.AddSingleton<IReachabilityUnionPublisherService, ReachabilityUnionPublisherService>();
builder.Services.AddSingleton<IScanStageExecutor, StellaOps.Scanner.Worker.Processing.Replay.ReplaySealedBundleStageExecutor>();
builder.Services.AddSingleton<StellaOps.Scanner.Worker.Processing.Replay.ReplayBundleFetcher>();
var storageSection = builder.Configuration.GetSection("ScannerStorage");
var connectionString = storageSection.GetValue<string>("Mongo:ConnectionString");

View File

@@ -25,6 +25,7 @@ using StellaOps.Scanner.Worker.Processing.Surface;
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
using Xunit;
using StellaOps.Cryptography;
using StellaOps.Scanner.Worker.Determinism;
namespace StellaOps.Scanner.Worker.Tests;
@@ -99,6 +100,7 @@ public sealed class SurfaceManifestStageExecutorTests
Assert.True(context.Analysis.TryGet<SurfaceManifestPublishResult>(ScanAnalysisKeys.SurfaceManifest, out var result));
Assert.NotNull(result);
Assert.Equal(publisher.LastManifestDigest, result!.ManifestDigest);
Assert.Equal(result.DeterminismMerkleRoot, publisher.LastRequest!.DeterminismMerkleRoot);
Assert.Equal(4, cache.Entries.Count);
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.entrytrace.graph" && key.Tenant == "tenant-a");
@@ -163,6 +165,10 @@ public sealed class SurfaceManifestStageExecutorTests
Assert.Equal("feed-001", json.RootElement.GetProperty("pins").GetProperty("feed").GetString());
Assert.Equal("rev-77", json.RootElement.GetProperty("pins").GetProperty("policy").GetString());
Assert.True(json.RootElement.GetProperty("artifacts").EnumerateObject().Any());
Assert.True(context.Analysis.TryGet<DeterminismEvidence>(ScanAnalysisKeys.DeterminismEvidence, out var evidence));
Assert.False(string.IsNullOrWhiteSpace(evidence!.MerkleRootSha256));
Assert.Equal(evidence.PayloadHashes["entrytrace.ndjson"], json.RootElement.GetProperty("artifacts").GetProperty("entrytrace.ndjson").GetString());
}
[Fact]
@@ -500,7 +506,8 @@ public sealed class SurfaceManifestStageExecutorTests
ManifestDigest: manifestDigest,
ManifestUri: $"cas://test/manifests/{manifestDigest}",
ArtifactId: $"surface-manifest::{manifestDigest}",
Document: document);
Document: document,
DeterminismMerkleRoot: request.DeterminismMerkleRoot);
return Task.FromResult(result);
}