up
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -16,6 +16,8 @@ public sealed class DeterminismContext
|
||||
ConcurrencyLimit = concurrencyLimit;
|
||||
}
|
||||
|
||||
public bool IsDeterminismEnabled => FixedClock || RngSeed.HasValue || ConcurrencyLimit.HasValue || FilterLogs;
|
||||
|
||||
public bool FixedClock { get; }
|
||||
|
||||
public DateTimeOffset FixedInstantUtc { get; }
|
||||
|
||||
@@ -74,6 +74,12 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
|
||||
|
||||
var payloads = CollectPayloads(context);
|
||||
await PersistRubyPackagesAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var determinismPayload = BuildDeterminismPayload(context, payloads, out var merkleRoot);
|
||||
if (determinismPayload is not null)
|
||||
{
|
||||
payloads.Add(determinismPayload);
|
||||
}
|
||||
if (payloads.Count == 0)
|
||||
{
|
||||
_metrics.RecordSurfaceManifestSkipped(context);
|
||||
@@ -96,7 +102,12 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
|
||||
Payloads: payloads,
|
||||
Component: "scanner.worker",
|
||||
Version: _componentVersion,
|
||||
WorkerInstance: Environment.MachineName);
|
||||
WorkerInstance: Environment.MachineName,
|
||||
DeterminismMerkleRoot: merkleRoot,
|
||||
ReplayBundleUri: GetReplayBundleUri(context),
|
||||
ReplayBundleHash: GetReplayBundleHash(context),
|
||||
ReplayPolicyPin: GetPin(context, "determinism.policy"),
|
||||
ReplayFeedPin: GetPin(context, "determinism.feed"));
|
||||
|
||||
var result = await _publisher.PublishAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -233,8 +244,9 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
|
||||
return payloads;
|
||||
}
|
||||
|
||||
private SurfaceManifestPayload? BuildDeterminismPayload(ScanJobContext context, IEnumerable<SurfaceManifestPayload> payloads)
|
||||
private SurfaceManifestPayload? BuildDeterminismPayload(ScanJobContext context, IEnumerable<SurfaceManifestPayload> payloads, out string? merkleRoot)
|
||||
{
|
||||
merkleRoot = null;
|
||||
var pins = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (context.Lease.Metadata.TryGetValue("determinism.feed", out var feed) && !string.IsNullOrWhiteSpace(feed))
|
||||
{
|
||||
@@ -246,12 +258,8 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
|
||||
pins["policy"] = policy;
|
||||
}
|
||||
|
||||
var artifactHashes = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var payload in payloads)
|
||||
{
|
||||
var digest = ComputeDigest(payload.Content.Span);
|
||||
artifactHashes[payload.Kind] = digest;
|
||||
}
|
||||
var (artifactHashes, merkle) = ComputeDeterminismHashes(payloads);
|
||||
merkleRoot = merkle;
|
||||
|
||||
var report = new
|
||||
{
|
||||
@@ -261,9 +269,13 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
|
||||
filterLogs = _determinism.FilterLogs,
|
||||
concurrencyLimit = _determinism.ConcurrencyLimit,
|
||||
pins = pins,
|
||||
artifacts = artifactHashes
|
||||
artifacts = artifactHashes,
|
||||
merkleRoot = merkle
|
||||
};
|
||||
|
||||
var evidence = new Determinism.DeterminismEvidence(artifactHashes, merkle);
|
||||
context.Analysis.Set(ScanAnalysisKeys.DeterminismEvidence, evidence);
|
||||
|
||||
var json = JsonSerializer.Serialize(report, JsonOptions);
|
||||
return new SurfaceManifestPayload(
|
||||
ArtifactDocumentType.SurfaceObservation,
|
||||
@@ -274,6 +286,46 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
|
||||
View: "replay");
|
||||
}
|
||||
|
||||
private static (Dictionary<string, string> Hashes, string MerkleRoot) ComputeDeterminismHashes(IEnumerable<SurfaceManifestPayload> payloads)
|
||||
{
|
||||
var map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
using var sha = SHA256.Create();
|
||||
|
||||
foreach (var payload in payloads.OrderBy(p => p.Kind, StringComparer.Ordinal))
|
||||
{
|
||||
var digest = ComputeDigest(payload.Content.Span);
|
||||
map[payload.Kind] = digest;
|
||||
}
|
||||
|
||||
// Build Merkle-like root by hashing the ordered list of kind:digest lines.
|
||||
var builder = new StringBuilder();
|
||||
foreach (var kvp in map.OrderBy(kv => kv.Key, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append(kvp.Key).Append(':').Append(kvp.Value).Append('\n');
|
||||
}
|
||||
|
||||
var rootBytes = Encoding.UTF8.GetBytes(builder.ToString());
|
||||
var rootHash = sha.ComputeHash(rootBytes);
|
||||
var merkleRoot = Convert.ToHexString(rootHash).ToLowerInvariant();
|
||||
|
||||
return (map, merkleRoot);
|
||||
}
|
||||
|
||||
private static string? GetReplayBundleUri(ScanJobContext context)
|
||||
=> context.Lease.Metadata.TryGetValue("replay.bundle.uri", out var value) && !string.IsNullOrWhiteSpace(value)
|
||||
? value.Trim()
|
||||
: null;
|
||||
|
||||
private static string? GetReplayBundleHash(ScanJobContext context)
|
||||
=> context.Lease.Metadata.TryGetValue("replay.bundle.sha256", out var value) && !string.IsNullOrWhiteSpace(value)
|
||||
? value.Trim().ToLowerInvariant()
|
||||
: null;
|
||||
|
||||
private static string? GetPin(ScanJobContext context, string key)
|
||||
=> context.Lease.Metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)
|
||||
? value.Trim()
|
||||
: null;
|
||||
|
||||
private async Task PersistRubyPackagesAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!context.Analysis.TryGet<ReadOnlyDictionary<string, LanguageAnalyzerResult>>(ScanAnalysisKeys.LanguageAnalyzerResults, out var results))
|
||||
|
||||
@@ -87,6 +87,8 @@ builder.Services.AddSingleton<IEntryTraceExecutionService, EntryTraceExecutionSe
|
||||
builder.Services.AddSingleton<ReachabilityUnionWriter>();
|
||||
builder.Services.AddSingleton<ReachabilityUnionPublisher>();
|
||||
builder.Services.AddSingleton<IReachabilityUnionPublisherService, ReachabilityUnionPublisherService>();
|
||||
builder.Services.AddSingleton<IScanStageExecutor, StellaOps.Scanner.Worker.Processing.Replay.ReplaySealedBundleStageExecutor>();
|
||||
builder.Services.AddSingleton<StellaOps.Scanner.Worker.Processing.Replay.ReplayBundleFetcher>();
|
||||
|
||||
var storageSection = builder.Configuration.GetSection("ScannerStorage");
|
||||
var connectionString = storageSection.GetValue<string>("Mongo:ConnectionString");
|
||||
|
||||
@@ -25,6 +25,7 @@ using StellaOps.Scanner.Worker.Processing.Surface;
|
||||
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
|
||||
using Xunit;
|
||||
using StellaOps.Cryptography;
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
@@ -99,6 +100,7 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
Assert.True(context.Analysis.TryGet<SurfaceManifestPublishResult>(ScanAnalysisKeys.SurfaceManifest, out var result));
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(publisher.LastManifestDigest, result!.ManifestDigest);
|
||||
Assert.Equal(result.DeterminismMerkleRoot, publisher.LastRequest!.DeterminismMerkleRoot);
|
||||
|
||||
Assert.Equal(4, cache.Entries.Count);
|
||||
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.entrytrace.graph" && key.Tenant == "tenant-a");
|
||||
@@ -163,6 +165,10 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
Assert.Equal("feed-001", json.RootElement.GetProperty("pins").GetProperty("feed").GetString());
|
||||
Assert.Equal("rev-77", json.RootElement.GetProperty("pins").GetProperty("policy").GetString());
|
||||
Assert.True(json.RootElement.GetProperty("artifacts").EnumerateObject().Any());
|
||||
|
||||
Assert.True(context.Analysis.TryGet<DeterminismEvidence>(ScanAnalysisKeys.DeterminismEvidence, out var evidence));
|
||||
Assert.False(string.IsNullOrWhiteSpace(evidence!.MerkleRootSha256));
|
||||
Assert.Equal(evidence.PayloadHashes["entrytrace.ndjson"], json.RootElement.GetProperty("artifacts").GetProperty("entrytrace.ndjson").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -500,7 +506,8 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
ManifestDigest: manifestDigest,
|
||||
ManifestUri: $"cas://test/manifests/{manifestDigest}",
|
||||
ArtifactId: $"surface-manifest::{manifestDigest}",
|
||||
Document: document);
|
||||
Document: document,
|
||||
DeterminismMerkleRoot: request.DeterminismMerkleRoot);
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user