292 lines
10 KiB
C#
292 lines
10 KiB
C#
using System.Collections.Immutable;
|
|
using System.Text.Json;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using StellaOps.Policy;
|
|
|
|
var scenarioRoot = "samples/policy/simulations";
|
|
string? outputDir = null;
|
|
|
|
for (var i = 0; i < args.Length; i++)
|
|
{
|
|
var arg = args[i];
|
|
switch (arg)
|
|
{
|
|
case "--scenario-root":
|
|
case "-r":
|
|
if (i + 1 >= args.Length)
|
|
{
|
|
Console.Error.WriteLine("Missing value for --scenario-root.");
|
|
return 64;
|
|
}
|
|
scenarioRoot = args[++i];
|
|
break;
|
|
case "--output":
|
|
case "-o":
|
|
if (i + 1 >= args.Length)
|
|
{
|
|
Console.Error.WriteLine("Missing value for --output.");
|
|
return 64;
|
|
}
|
|
outputDir = args[++i];
|
|
break;
|
|
case "--help":
|
|
case "-h":
|
|
case "-?":
|
|
PrintUsage();
|
|
return 0;
|
|
default:
|
|
Console.Error.WriteLine($"Unknown argument '{arg}'.");
|
|
PrintUsage();
|
|
return 64;
|
|
}
|
|
}
|
|
|
|
if (!Directory.Exists(scenarioRoot))
|
|
{
|
|
Console.Error.WriteLine($"Scenario root '{scenarioRoot}' does not exist.");
|
|
return 66;
|
|
}
|
|
|
|
var scenarioFiles = Directory.GetFiles(scenarioRoot, "scenario.json", SearchOption.AllDirectories);
|
|
if (scenarioFiles.Length == 0)
|
|
{
|
|
Console.Error.WriteLine($"No scenario.json files found under '{scenarioRoot}'.");
|
|
return 0;
|
|
}
|
|
|
|
var loggerFactory = NullLoggerFactory.Instance;
|
|
var snapshotStore = new PolicySnapshotStore(
|
|
new NullPolicySnapshotRepository(),
|
|
new NullPolicyAuditRepository(),
|
|
TimeProvider.System,
|
|
loggerFactory.CreateLogger<PolicySnapshotStore>());
|
|
var previewService = new PolicyPreviewService(snapshotStore, loggerFactory.CreateLogger<PolicyPreviewService>());
|
|
|
|
var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
|
{
|
|
PropertyNameCaseInsensitive = true,
|
|
ReadCommentHandling = JsonCommentHandling.Skip,
|
|
};
|
|
|
|
var summary = new List<ScenarioResult>();
|
|
var success = true;
|
|
|
|
foreach (var scenarioFile in scenarioFiles.OrderBy(static f => f, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
var scenarioText = await File.ReadAllTextAsync(scenarioFile);
|
|
var scenario = JsonSerializer.Deserialize<PolicySimulationScenario>(scenarioText, serializerOptions);
|
|
if (scenario is null)
|
|
{
|
|
Console.Error.WriteLine($"Failed to deserialize scenario '{scenarioFile}'.");
|
|
success = false;
|
|
continue;
|
|
}
|
|
|
|
var repoRoot = Directory.GetCurrentDirectory();
|
|
var policyPath = Path.Combine(repoRoot, scenario.PolicyPath);
|
|
if (!File.Exists(policyPath))
|
|
{
|
|
Console.Error.WriteLine($"Policy file '{scenario.PolicyPath}' referenced by scenario '{scenario.Name}' does not exist.");
|
|
success = false;
|
|
continue;
|
|
}
|
|
|
|
var policyContent = await File.ReadAllTextAsync(policyPath);
|
|
var policyFormat = PolicySchema.DetectFormat(policyPath);
|
|
var findings = scenario.Findings.Select(ToPolicyFinding).ToImmutableArray();
|
|
var baseline = scenario.Baseline?.Select(ToPolicyVerdict).ToImmutableArray() ?? ImmutableArray<PolicyVerdict>.Empty;
|
|
|
|
var request = new PolicyPreviewRequest(
|
|
ImageDigest: $"sha256:simulation-{scenario.Name}",
|
|
Findings: findings,
|
|
BaselineVerdicts: baseline,
|
|
SnapshotOverride: null,
|
|
ProposedPolicy: new PolicySnapshotContent(
|
|
Content: policyContent,
|
|
Format: policyFormat,
|
|
Actor: "ci",
|
|
Source: "ci/simulation-smoke",
|
|
Description: $"CI simulation for scenario '{scenario.Name}'"));
|
|
|
|
var response = await previewService.PreviewAsync(request, CancellationToken.None);
|
|
var scenarioResult = EvaluateScenario(scenario, response);
|
|
summary.Add(scenarioResult);
|
|
|
|
if (!scenarioResult.Success)
|
|
{
|
|
success = false;
|
|
}
|
|
}
|
|
|
|
if (outputDir is not null)
|
|
{
|
|
Directory.CreateDirectory(outputDir);
|
|
var summaryPath = Path.Combine(outputDir, "policy-simulation-summary.json");
|
|
await File.WriteAllTextAsync(summaryPath, JsonSerializer.Serialize(summary, new JsonSerializerOptions { WriteIndented = true }));
|
|
}
|
|
|
|
return success ? 0 : 1;
|
|
|
|
static void PrintUsage()
|
|
{
|
|
Console.WriteLine("Usage: policy-simulation-smoke [--scenario-root <path>] [--output <dir>]");
|
|
Console.WriteLine("Example: policy-simulation-smoke --scenario-root samples/policy/simulations --output artifacts/policy-simulations");
|
|
}
|
|
|
|
static PolicyFinding ToPolicyFinding(ScenarioFinding finding)
|
|
{
|
|
var tags = finding.Tags is null ? ImmutableArray<string>.Empty : ImmutableArray.CreateRange(finding.Tags);
|
|
var severity = Enum.Parse<PolicySeverity>(finding.Severity, ignoreCase: true);
|
|
return new PolicyFinding(
|
|
finding.FindingId,
|
|
severity,
|
|
finding.Environment,
|
|
finding.Source,
|
|
finding.Vendor,
|
|
finding.License,
|
|
finding.Image,
|
|
finding.Repository,
|
|
finding.Package,
|
|
finding.Purl,
|
|
finding.Cve,
|
|
finding.Path,
|
|
finding.LayerDigest,
|
|
tags);
|
|
}
|
|
|
|
static PolicyVerdict ToPolicyVerdict(ScenarioBaseline baseline)
|
|
{
|
|
var status = Enum.Parse<PolicyVerdictStatus>(baseline.Status, ignoreCase: true);
|
|
var inputs = baseline.Inputs?.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase) ?? ImmutableDictionary<string, double>.Empty;
|
|
return new PolicyVerdict(
|
|
baseline.FindingId,
|
|
status,
|
|
RuleName: baseline.RuleName,
|
|
RuleAction: baseline.RuleAction,
|
|
Notes: baseline.Notes,
|
|
Score: baseline.Score,
|
|
ConfigVersion: baseline.ConfigVersion ?? PolicyScoringConfig.Default.Version,
|
|
Inputs: inputs,
|
|
QuietedBy: null,
|
|
Quiet: false,
|
|
UnknownConfidence: null,
|
|
ConfidenceBand: null,
|
|
UnknownAgeDays: null,
|
|
SourceTrust: null,
|
|
Reachability: null);
|
|
}
|
|
|
|
static ScenarioResult EvaluateScenario(PolicySimulationScenario scenario, PolicyPreviewResponse response)
|
|
{
|
|
var result = new ScenarioResult(scenario.Name);
|
|
if (!response.Success)
|
|
{
|
|
result.Failures.Add("Preview failed.");
|
|
return result with { Success = false, ChangedCount = response.ChangedCount };
|
|
}
|
|
|
|
var diffs = response.Diffs.ToDictionary(diff => diff.Projected.FindingId, StringComparer.OrdinalIgnoreCase);
|
|
foreach (var expected in scenario.ExpectedDiffs)
|
|
{
|
|
if (!diffs.TryGetValue(expected.FindingId, out var diff))
|
|
{
|
|
result.Failures.Add($"Expected finding '{expected.FindingId}' missing from diff.");
|
|
continue;
|
|
}
|
|
|
|
var projectedStatus = diff.Projected.Status.ToString();
|
|
result.ActualStatuses[expected.FindingId] = projectedStatus;
|
|
if (!string.Equals(projectedStatus, expected.Status, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
result.Failures.Add($"Finding '{expected.FindingId}' expected status '{expected.Status}' but was '{projectedStatus}'.");
|
|
}
|
|
}
|
|
|
|
foreach (var diff in diffs.Values)
|
|
{
|
|
if (!result.ActualStatuses.ContainsKey(diff.Projected.FindingId))
|
|
{
|
|
result.ActualStatuses[diff.Projected.FindingId] = diff.Projected.Status.ToString();
|
|
}
|
|
}
|
|
|
|
var success = result.Failures.Count == 0;
|
|
return result with
|
|
{
|
|
Success = success,
|
|
ChangedCount = response.ChangedCount
|
|
};
|
|
}
|
|
|
|
internal sealed record PolicySimulationScenario
|
|
{
|
|
public string Name { get; init; } = "scenario";
|
|
public string PolicyPath { get; init; } = string.Empty;
|
|
public List<ScenarioFinding> Findings { get; init; } = new();
|
|
public List<ScenarioExpectedDiff> ExpectedDiffs { get; init; } = new();
|
|
public List<ScenarioBaseline>? Baseline { get; init; }
|
|
}
|
|
|
|
internal sealed record ScenarioFinding
|
|
{
|
|
public string FindingId { get; init; } = string.Empty;
|
|
public string Severity { get; init; } = "Low";
|
|
public string? Environment { get; init; }
|
|
public string? Source { get; init; }
|
|
public string? Vendor { get; init; }
|
|
public string? License { get; init; }
|
|
public string? Image { get; init; }
|
|
public string? Repository { get; init; }
|
|
public string? Package { get; init; }
|
|
public string? Purl { get; init; }
|
|
public string? Cve { get; init; }
|
|
public string? Path { get; init; }
|
|
public string? LayerDigest { get; init; }
|
|
public string[]? Tags { get; init; }
|
|
}
|
|
|
|
internal sealed record ScenarioExpectedDiff
|
|
{
|
|
public string FindingId { get; init; } = string.Empty;
|
|
public string Status { get; init; } = "Pass";
|
|
}
|
|
|
|
internal sealed record ScenarioBaseline
|
|
{
|
|
public string FindingId { get; init; } = string.Empty;
|
|
public string Status { get; init; } = "Pass";
|
|
public string? RuleName { get; init; }
|
|
public string? RuleAction { get; init; }
|
|
public string? Notes { get; init; }
|
|
public double Score { get; init; }
|
|
public string? ConfigVersion { get; init; }
|
|
public Dictionary<string, double>? Inputs { get; init; }
|
|
}
|
|
|
|
internal sealed record ScenarioResult(string ScenarioName)
|
|
{
|
|
public bool Success { get; init; } = true;
|
|
public int ChangedCount { get; init; }
|
|
public List<string> Failures { get; } = new();
|
|
public Dictionary<string, string> ActualStatuses { get; } = new(StringComparer.OrdinalIgnoreCase);
|
|
}
|
|
|
|
internal sealed class NullPolicySnapshotRepository : IPolicySnapshotRepository
|
|
{
|
|
public Task AddAsync(PolicySnapshot snapshot, CancellationToken cancellationToken = default) => Task.CompletedTask;
|
|
|
|
public Task<PolicySnapshot?> GetLatestAsync(CancellationToken cancellationToken = default) => Task.FromResult<PolicySnapshot?>(null);
|
|
|
|
public Task<IReadOnlyList<PolicySnapshot>> ListAsync(int limit, CancellationToken cancellationToken = default)
|
|
=> Task.FromResult<IReadOnlyList<PolicySnapshot>>(Array.Empty<PolicySnapshot>());
|
|
}
|
|
|
|
internal sealed class NullPolicyAuditRepository : IPolicyAuditRepository
|
|
{
|
|
public Task AddAsync(PolicyAuditEntry entry, CancellationToken cancellationToken = default) => Task.CompletedTask;
|
|
|
|
public Task<IReadOnlyList<PolicyAuditEntry>> ListAsync(int limit, CancellationToken cancellationToken = default)
|
|
=> Task.FromResult<IReadOnlyList<PolicyAuditEntry>>(Array.Empty<PolicyAuditEntry>());
|
|
}
|