feat: Add VEX compact fixture and implement offline verifier for Findings Ledger exports
- Introduced a new VEX compact fixture for testing purposes. - Implemented `verify_export.py` script to validate Findings Ledger exports, ensuring deterministic ordering and applying redaction manifests. - Added a lightweight stub `HarnessRunner` for unit tests to validate ledger hashing expectations. - Documented tasks related to the Mirror Creator. - Created models for entropy signals and implemented the `EntropyPenaltyCalculator` to compute penalties based on scanner outputs. - Developed unit tests for `EntropyPenaltyCalculator` to ensure correct penalty calculations and handling of edge cases. - Added tests for symbol ID normalization in the reachability scanner. - Enhanced console status service with comprehensive unit tests for connection handling and error recovery. - Included Cosign tool version 2.6.0 with checksums for various platforms.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,143 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Signals.Entropy;
|
||||
|
||||
/// <summary>
|
||||
/// Summary of opaque ratios per image layer emitted by the scanner.
|
||||
/// </summary>
|
||||
public sealed class EntropyLayerSummary
|
||||
{
|
||||
[JsonPropertyName("schema")]
|
||||
public string? Schema { get; init; }
|
||||
|
||||
[JsonPropertyName("generatedAt")]
|
||||
public DateTimeOffset? GeneratedAt { get; init; }
|
||||
|
||||
[JsonPropertyName("imageDigest")]
|
||||
public string? ImageDigest { get; init; }
|
||||
|
||||
[JsonPropertyName("layers")]
|
||||
public List<EntropyLayer> Layers { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("imageOpaqueRatio")]
|
||||
public decimal? ImageOpaqueRatio { get; init; }
|
||||
|
||||
[JsonPropertyName("entropyPenalty")]
|
||||
public decimal? EntropyPenalty { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Layer-level entropy ratios.
|
||||
/// </summary>
|
||||
public sealed class EntropyLayer
|
||||
{
|
||||
[JsonPropertyName("digest")]
|
||||
public string? Digest { get; init; }
|
||||
|
||||
[JsonPropertyName("opaqueBytes")]
|
||||
public long OpaqueBytes { get; init; }
|
||||
|
||||
[JsonPropertyName("totalBytes")]
|
||||
public long TotalBytes { get; init; }
|
||||
|
||||
[JsonPropertyName("opaqueRatio")]
|
||||
public decimal? OpaqueRatio { get; init; }
|
||||
|
||||
[JsonPropertyName("indicators")]
|
||||
public List<string> Indicators { get; init; } = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detailed entropy report for files within a layer.
|
||||
/// </summary>
|
||||
public sealed class EntropyReport
|
||||
{
|
||||
[JsonPropertyName("schema")]
|
||||
public string? Schema { get; init; }
|
||||
|
||||
[JsonPropertyName("generatedAt")]
|
||||
public DateTimeOffset? GeneratedAt { get; init; }
|
||||
|
||||
[JsonPropertyName("imageDigest")]
|
||||
public string? ImageDigest { get; init; }
|
||||
|
||||
[JsonPropertyName("layerDigest")]
|
||||
public string? LayerDigest { get; init; }
|
||||
|
||||
[JsonPropertyName("files")]
|
||||
public List<EntropyFile> Files { get; init; } = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-file entropy metrics.
|
||||
/// </summary>
|
||||
public sealed class EntropyFile
|
||||
{
|
||||
[JsonPropertyName("path")]
|
||||
public string Path { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("size")]
|
||||
public long Size { get; init; }
|
||||
|
||||
[JsonPropertyName("opaqueBytes")]
|
||||
public long OpaqueBytes { get; init; }
|
||||
|
||||
[JsonPropertyName("opaqueRatio")]
|
||||
public decimal? OpaqueRatio { get; init; }
|
||||
|
||||
[JsonPropertyName("flags")]
|
||||
public List<string> Flags { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("windows")]
|
||||
public List<EntropyWindow> Windows { get; init; } = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sliding window entropy value.
|
||||
/// </summary>
|
||||
public sealed class EntropyWindow
|
||||
{
|
||||
[JsonPropertyName("offset")]
|
||||
public long Offset { get; init; }
|
||||
|
||||
[JsonPropertyName("length")]
|
||||
public int Length { get; init; }
|
||||
|
||||
[JsonPropertyName("entropy")]
|
||||
public decimal Entropy { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computed entropy penalty result for policy trust algebra.
|
||||
/// </summary>
|
||||
public sealed record EntropyPenaltyResult(
|
||||
decimal Penalty,
|
||||
decimal RawPenalty,
|
||||
bool Capped,
|
||||
bool Blocked,
|
||||
bool Warned,
|
||||
decimal ImageOpaqueRatio,
|
||||
IReadOnlyList<EntropyLayerContribution> LayerContributions,
|
||||
IReadOnlyList<EntropyTopFile> TopFiles,
|
||||
IReadOnlyList<string> ReasonCodes,
|
||||
bool ProvenanceAttested);
|
||||
|
||||
/// <summary>
|
||||
/// Contribution of a layer to the final penalty.
|
||||
/// </summary>
|
||||
public sealed record EntropyLayerContribution(
|
||||
string LayerDigest,
|
||||
decimal OpaqueRatio,
|
||||
decimal Contribution,
|
||||
bool Mitigated,
|
||||
IReadOnlyList<string> Indicators);
|
||||
|
||||
/// <summary>
|
||||
/// Highest-entropy files for explanations.
|
||||
/// </summary>
|
||||
public sealed record EntropyTopFile(
|
||||
string Path,
|
||||
decimal OpaqueRatio,
|
||||
long OpaqueBytes,
|
||||
long Size,
|
||||
IReadOnlyList<string> Flags);
|
||||
@@ -0,0 +1,280 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Policy.Engine.Options;
|
||||
using StellaOps.Policy.Engine.Telemetry;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Signals.Entropy;
|
||||
|
||||
/// <summary>
|
||||
/// Computes entropy penalties from scanner outputs (`entropy.report.json`, `layer_summary.json`)
|
||||
/// and maps them into trust-algebra friendly signals.
|
||||
/// </summary>
|
||||
public sealed class EntropyPenaltyCalculator
|
||||
{
|
||||
private readonly PolicyEngineEntropyOptions _options;
|
||||
private readonly ILogger<EntropyPenaltyCalculator> _logger;
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
public EntropyPenaltyCalculator(
|
||||
IOptions<PolicyEngineOptions> options,
|
||||
ILogger<EntropyPenaltyCalculator> logger)
|
||||
{
|
||||
_options = options?.Value?.Entropy ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compute an entropy penalty from JSON payloads.
|
||||
/// </summary>
|
||||
/// <param name="layerSummaryJson">Contents of `layer_summary.json`.</param>
|
||||
/// <param name="entropyReportJson">Optional contents of `entropy.report.json`.</param>
|
||||
/// <param name="provenanceAttested">Whether provenance for the image is attested.</param>
|
||||
public EntropyPenaltyResult ComputeFromJson(
|
||||
string layerSummaryJson,
|
||||
string? entropyReportJson = null,
|
||||
bool provenanceAttested = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(layerSummaryJson))
|
||||
{
|
||||
throw new ArgumentException("layerSummaryJson is required", nameof(layerSummaryJson));
|
||||
}
|
||||
|
||||
var summary = Deserialize<EntropyLayerSummary>(layerSummaryJson)
|
||||
?? throw new InvalidOperationException("Failed to parse layer_summary.json");
|
||||
|
||||
EntropyReport? report = null;
|
||||
if (!string.IsNullOrWhiteSpace(entropyReportJson))
|
||||
{
|
||||
report = Deserialize<EntropyReport>(entropyReportJson);
|
||||
}
|
||||
|
||||
return Compute(summary, report, provenanceAttested);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compute an entropy penalty from deserialized models.
|
||||
/// </summary>
|
||||
public EntropyPenaltyResult Compute(
|
||||
EntropyLayerSummary summary,
|
||||
EntropyReport? report = null,
|
||||
bool provenanceAttested = false)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(summary);
|
||||
|
||||
var layers = summary.Layers ?? new List<EntropyLayer>();
|
||||
var orderedLayers = layers
|
||||
.OrderBy(l => l.Digest ?? string.Empty, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
var imageBytes = orderedLayers.Sum(l => Math.Max(0m, (decimal)l.TotalBytes));
|
||||
var imageOpaqueRatio = summary.ImageOpaqueRatio ?? ComputeImageOpaqueRatio(orderedLayers, imageBytes);
|
||||
|
||||
var contributions = new List<EntropyLayerContribution>(orderedLayers.Count);
|
||||
decimal contributionSum = 0m;
|
||||
var reasonCodes = new List<string>();
|
||||
var anyMitigated = false;
|
||||
|
||||
foreach (var layer in orderedLayers)
|
||||
{
|
||||
var indicators = NormalizeIndicators(layer.Indicators);
|
||||
var layerRatio = ResolveOpaqueRatio(layer);
|
||||
var layerWeight = imageBytes > 0 ? SafeDivide(layer.TotalBytes, imageBytes) : 0m;
|
||||
var mitigated = provenanceAttested && HasSymbols(indicators);
|
||||
var effectiveRatio = mitigated ? layerRatio * _options.SymbolMitigationFactor : layerRatio;
|
||||
var contribution = Math.Round(effectiveRatio * layerWeight, 6, MidpointRounding.ToZero);
|
||||
|
||||
contributions.Add(new EntropyLayerContribution(
|
||||
layer.Digest ?? "unknown",
|
||||
layerRatio,
|
||||
contribution,
|
||||
mitigated,
|
||||
indicators));
|
||||
|
||||
contributionSum += contribution;
|
||||
anyMitigated |= mitigated;
|
||||
}
|
||||
|
||||
var rawPenalty = Math.Round(contributionSum * _options.PenaltyMultiplier, 6, MidpointRounding.ToZero);
|
||||
var cappedPenalty = Math.Min(_options.PenaltyCap, rawPenalty);
|
||||
var penalty = Math.Round(cappedPenalty, 4, MidpointRounding.ToZero);
|
||||
var capped = rawPenalty > _options.PenaltyCap;
|
||||
|
||||
var warnTriggered = !string.IsNullOrWhiteSpace(FindFirstWarnReason(orderedLayers, report));
|
||||
var blocked = imageOpaqueRatio > _options.ImageOpaqueBlockThreshold && !provenanceAttested;
|
||||
var warn = !blocked && warnTriggered;
|
||||
|
||||
var topFiles = BuildTopFiles(report);
|
||||
|
||||
PopulateReasonCodes(
|
||||
reasonCodes,
|
||||
imageOpaqueRatio,
|
||||
orderedLayers,
|
||||
report,
|
||||
capped,
|
||||
anyMitigated,
|
||||
provenanceAttested,
|
||||
blocked,
|
||||
warn);
|
||||
|
||||
PolicyEngineTelemetry.RecordEntropyPenalty(
|
||||
(double)penalty,
|
||||
blocked ? "block" : warn ? "warn" : "ok",
|
||||
(double)imageOpaqueRatio,
|
||||
topFiles.Count > 0 ? (double?)topFiles[0].OpaqueRatio : null);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Computed entropy penalty {Penalty:F4} (raw {Raw:F4}, imageOpaqueRatio={ImageOpaqueRatio:F3}, blocked={Blocked}, warn={Warn}, capped={Capped})",
|
||||
penalty,
|
||||
rawPenalty,
|
||||
imageOpaqueRatio,
|
||||
blocked,
|
||||
warn,
|
||||
capped);
|
||||
|
||||
return new EntropyPenaltyResult(
|
||||
Penalty: penalty,
|
||||
RawPenalty: rawPenalty,
|
||||
Capped: capped,
|
||||
Blocked: blocked,
|
||||
Warned: warn,
|
||||
ImageOpaqueRatio: imageOpaqueRatio,
|
||||
LayerContributions: contributions,
|
||||
TopFiles: topFiles,
|
||||
ReasonCodes: reasonCodes,
|
||||
ProvenanceAttested: provenanceAttested);
|
||||
}
|
||||
|
||||
private static decimal ComputeImageOpaqueRatio(IEnumerable<EntropyLayer> layers, decimal imageBytes)
|
||||
{
|
||||
if (imageBytes <= 0m)
|
||||
{
|
||||
return 0m;
|
||||
}
|
||||
|
||||
var opaqueBytes = layers.Sum(l => Math.Max(0m, (decimal)l.OpaqueBytes));
|
||||
return Math.Round(SafeDivide(opaqueBytes, imageBytes), 6, MidpointRounding.ToZero);
|
||||
}
|
||||
|
||||
private static decimal ResolveOpaqueRatio(EntropyLayer layer)
|
||||
{
|
||||
if (layer.TotalBytes > 0)
|
||||
{
|
||||
return Math.Round(SafeDivide(layer.OpaqueBytes, layer.TotalBytes), 6, MidpointRounding.ToZero);
|
||||
}
|
||||
|
||||
return Math.Max(0m, layer.OpaqueRatio ?? 0m);
|
||||
}
|
||||
|
||||
private static decimal SafeDivide(decimal numerator, decimal denominator)
|
||||
=> denominator <= 0 ? 0 : numerator / denominator;
|
||||
|
||||
private static bool HasSymbols(IReadOnlyCollection<string> indicators)
|
||||
{
|
||||
return indicators.Any(i =>
|
||||
i.Equals("symbols", StringComparison.OrdinalIgnoreCase) ||
|
||||
i.Equals("has-symbols", StringComparison.OrdinalIgnoreCase) ||
|
||||
i.Equals("debug-symbols", StringComparison.OrdinalIgnoreCase) ||
|
||||
i.Equals("symbols-present", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeIndicators(IEnumerable<string> indicators)
|
||||
{
|
||||
return indicators
|
||||
.Where(indicator => !string.IsNullOrWhiteSpace(indicator))
|
||||
.Select(indicator => indicator.Trim())
|
||||
.OrderBy(indicator => indicator, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private IReadOnlyList<EntropyTopFile> BuildTopFiles(EntropyReport? report)
|
||||
{
|
||||
if (report?.Files == null || report.Files.Count == 0)
|
||||
{
|
||||
return Array.Empty<EntropyTopFile>();
|
||||
}
|
||||
|
||||
return report.Files
|
||||
.Where(f => f.OpaqueRatio.HasValue)
|
||||
.OrderByDescending(f => f.OpaqueRatio ?? 0m)
|
||||
.ThenByDescending(f => f.OpaqueBytes)
|
||||
.ThenBy(f => f.Path, StringComparer.Ordinal)
|
||||
.Take(_options.TopFiles)
|
||||
.Select(f => new EntropyTopFile(
|
||||
Path: f.Path,
|
||||
OpaqueRatio: Math.Round(f.OpaqueRatio ?? 0m, 6, MidpointRounding.ToZero),
|
||||
OpaqueBytes: f.OpaqueBytes,
|
||||
Size: f.Size,
|
||||
Flags: NormalizeIndicators(f.Flags)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private string? FindFirstWarnReason(IEnumerable<EntropyLayer> layers, EntropyReport? report)
|
||||
{
|
||||
var layerHit = layers.Any(l => ResolveOpaqueRatio(l) > _options.FileOpaqueWarnThreshold);
|
||||
if (layerHit)
|
||||
{
|
||||
return "layer_opaque_ratio";
|
||||
}
|
||||
|
||||
if (report?.Files is { Count: > 0 })
|
||||
{
|
||||
var fileHit = report.Files.Any(f => (f.OpaqueRatio ?? 0m) > _options.FileOpaqueWarnThreshold);
|
||||
if (fileHit)
|
||||
{
|
||||
return "file_opaque_ratio";
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private void PopulateReasonCodes(
|
||||
List<string> reasons,
|
||||
decimal imageOpaqueRatio,
|
||||
IReadOnlyCollection<EntropyLayer> layers,
|
||||
EntropyReport? report,
|
||||
bool capped,
|
||||
bool mitigated,
|
||||
bool provenanceAttested,
|
||||
bool blocked,
|
||||
bool warn)
|
||||
{
|
||||
if (blocked)
|
||||
{
|
||||
reasons.Add("image_opaque_ratio_exceeds_threshold");
|
||||
if (!provenanceAttested)
|
||||
{
|
||||
reasons.Add("provenance_unknown");
|
||||
}
|
||||
}
|
||||
|
||||
if (warn)
|
||||
{
|
||||
reasons.Add("file_opaque_ratio_exceeds_threshold");
|
||||
}
|
||||
|
||||
if (capped)
|
||||
{
|
||||
reasons.Add("penalty_capped");
|
||||
}
|
||||
|
||||
if (mitigated && provenanceAttested)
|
||||
{
|
||||
reasons.Add("symbols_mitigated");
|
||||
}
|
||||
|
||||
if (imageOpaqueRatio <= 0 && layers.Count == 0 && (report?.Files.Count ?? 0) == 0)
|
||||
{
|
||||
reasons.Add("no_entropy_data");
|
||||
}
|
||||
}
|
||||
|
||||
private static T? Deserialize<T>(string json)
|
||||
{
|
||||
return JsonSerializer.Deserialize<T>(json, JsonOptions);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user