git.stella-ops.org/tools/PolicySimulationSmoke/Program.cs

using System.Collections.Immutable;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Policy;

var scenarioRoot = "samples/policy/simulations";
string? outputDir = null;

for (var i = 0; i < args.Length; i++)
{
    var arg = args[i];
    switch (arg)
    {
        case "--scenario-root":
        case "-r":
            if (i + 1 >= args.Length)
            {
                Console.Error.WriteLine("Missing value for --scenario-root.");
                return 64;
            }
            scenarioRoot = args[++i];
            break;
        case "--output":
        case "-o":
            if (i + 1 >= args.Length)
            {
                Console.Error.WriteLine("Missing value for --output.");
                return 64;
            }
            outputDir = args[++i];
            break;
        case "--help":
        case "-h":
        case "-?":
            PrintUsage();
            return 0;
        default:
            Console.Error.WriteLine($"Unknown argument '{arg}'.");
            PrintUsage();
            return 64;
    }
}

if (!Directory.Exists(scenarioRoot))
{
    Console.Error.WriteLine($"Scenario root '{scenarioRoot}' does not exist.");
    return 66;
}

var scenarioFiles = Directory.GetFiles(scenarioRoot, "scenario.json", SearchOption.AllDirectories);
if (scenarioFiles.Length == 0)
{
    Console.Error.WriteLine($"No scenario.json files found under '{scenarioRoot}'.");
    return 0;
}

var loggerFactory = NullLoggerFactory.Instance;
var snapshotStore = new PolicySnapshotStore(
    new NullPolicySnapshotRepository(),
    new NullPolicyAuditRepository(),
    TimeProvider.System,
    loggerFactory.CreateLogger<PolicySnapshotStore>());
var previewService = new PolicyPreviewService(snapshotStore, loggerFactory.CreateLogger<PolicyPreviewService>());

var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
{
    PropertyNameCaseInsensitive = true,
    ReadCommentHandling = JsonCommentHandling.Skip,
};

var summary = new List<ScenarioResult>();
var success = true;

foreach (var scenarioFile in scenarioFiles.OrderBy(static f => f, StringComparer.OrdinalIgnoreCase))
{
    var scenarioText = await File.ReadAllTextAsync(scenarioFile);
    var scenario = JsonSerializer.Deserialize<PolicySimulationScenario>(scenarioText, serializerOptions);
    if (scenario is null)
    {
        Console.Error.WriteLine($"Failed to deserialize scenario '{scenarioFile}'.");
        success = false;
        continue;
    }

    var repoRoot = Directory.GetCurrentDirectory();
    var policyPath = Path.Combine(repoRoot, scenario.PolicyPath);
    if (!File.Exists(policyPath))
    {
        Console.Error.WriteLine($"Policy file '{scenario.PolicyPath}' referenced by scenario '{scenario.Name}' does not exist.");
        success = false;
        continue;
    }

    var policyContent = await File.ReadAllTextAsync(policyPath);
    var policyFormat = PolicySchema.DetectFormat(policyPath);
    var findings = scenario.Findings.Select(ToPolicyFinding).ToImmutableArray();
    var baseline = scenario.Baseline?.Select(ToPolicyVerdict).ToImmutableArray() ?? ImmutableArray<PolicyVerdict>.Empty;

    var request = new PolicyPreviewRequest(
        ImageDigest: $"sha256:simulation-{scenario.Name}",
        Findings: findings,
        BaselineVerdicts: baseline,
        SnapshotOverride: null,
        ProposedPolicy: new PolicySnapshotContent(
            Content: policyContent,
            Format: policyFormat,
            Actor: "ci",
            Source: "ci/simulation-smoke",
            Description: $"CI simulation for scenario '{scenario.Name}'"));

    var response = await previewService.PreviewAsync(request, CancellationToken.None);
    var scenarioResult = EvaluateScenario(scenario, response);
    summary.Add(scenarioResult);

    if (!scenarioResult.Success)
    {
        success = false;
    }
}

if (outputDir is not null)
{
    Directory.CreateDirectory(outputDir);
    var summaryPath = Path.Combine(outputDir, "policy-simulation-summary.json");
    await File.WriteAllTextAsync(summaryPath, JsonSerializer.Serialize(summary, new JsonSerializerOptions { WriteIndented = true }));
}

return success ? 0 : 1;

static void PrintUsage()
{
    Console.WriteLine("Usage: policy-simulation-smoke [--scenario-root <path>] [--output <dir>]");
    Console.WriteLine("Example: policy-simulation-smoke --scenario-root samples/policy/simulations --output artifacts/policy-simulations");
}

static PolicyFinding ToPolicyFinding(ScenarioFinding finding)
{
    var tags = finding.Tags is null ? ImmutableArray<string>.Empty : ImmutableArray.CreateRange(finding.Tags);
    var severity = Enum.Parse<PolicySeverity>(finding.Severity, ignoreCase: true);
    return new PolicyFinding(
        finding.FindingId,
        severity,
        finding.Environment,
        finding.Source,
        finding.Vendor,
        finding.License,
        finding.Image,
        finding.Repository,
        finding.Package,
        finding.Purl,
        finding.Cve,
        finding.Path,
        finding.LayerDigest,
        tags);
}

static PolicyVerdict ToPolicyVerdict(ScenarioBaseline baseline)
{
    var status = Enum.Parse<PolicyVerdictStatus>(baseline.Status, ignoreCase: true);
    var inputs = baseline.Inputs?.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase) ?? ImmutableDictionary<string, double>.Empty;
    return new PolicyVerdict(
        baseline.FindingId,
        status,
        RuleName: baseline.RuleName,
        RuleAction: baseline.RuleAction,
        Notes: baseline.Notes,
        Score: baseline.Score,
        ConfigVersion: baseline.ConfigVersion ?? PolicyScoringConfig.Default.Version,
        Inputs: inputs,
        QuietedBy: null,
        Quiet: false,
        UnknownConfidence: null,
        ConfidenceBand: null,
        UnknownAgeDays: null,
        SourceTrust: null,
        Reachability: null);
}

static ScenarioResult EvaluateScenario(PolicySimulationScenario scenario, PolicyPreviewResponse response)
{
    var result = new ScenarioResult(scenario.Name);
    if (!response.Success)
    {
        result.Failures.Add("Preview failed.");
        return result with { Success = false, ChangedCount = response.ChangedCount };
    }

    var diffs = response.Diffs.ToDictionary(diff => diff.Projected.FindingId, StringComparer.OrdinalIgnoreCase);
    foreach (var expected in scenario.ExpectedDiffs)
    {
        if (!diffs.TryGetValue(expected.FindingId, out var diff))
        {
            result.Failures.Add($"Expected finding '{expected.FindingId}' missing from diff.");
            continue;
        }

        var projectedStatus = diff.Projected.Status.ToString();
        result.ActualStatuses[expected.FindingId] = projectedStatus;
        if (!string.Equals(projectedStatus, expected.Status, StringComparison.OrdinalIgnoreCase))
        {
            result.Failures.Add($"Finding '{expected.FindingId}' expected status '{expected.Status}' but was '{projectedStatus}'.");
        }
    }

    foreach (var diff in diffs.Values)
    {
        if (!result.ActualStatuses.ContainsKey(diff.Projected.FindingId))
        {
            result.ActualStatuses[diff.Projected.FindingId] = diff.Projected.Status.ToString();
        }
    }

    var success = result.Failures.Count == 0;
    return result with
    {
        Success = success,
        ChangedCount = response.ChangedCount
    };
}

internal sealed record PolicySimulationScenario
{
    public string Name { get; init; } = "scenario";
    public string PolicyPath { get; init; } = string.Empty;
    public List<ScenarioFinding> Findings { get; init; } = new();
    public List<ScenarioExpectedDiff> ExpectedDiffs { get; init; } = new();
    public List<ScenarioBaseline>? Baseline { get; init; }
}

internal sealed record ScenarioFinding
{
    public string FindingId { get; init; } = string.Empty;
    public string Severity { get; init; } = "Low";
    public string? Environment { get; init; }
    public string? Source { get; init; }
    public string? Vendor { get; init; }
    public string? License { get; init; }
    public string? Image { get; init; }
    public string? Repository { get; init; }
    public string? Package { get; init; }
    public string? Purl { get; init; }
    public string? Cve { get; init; }
    public string? Path { get; init; }
    public string? LayerDigest { get; init; }
    public string[]? Tags { get; init; }
}

internal sealed record ScenarioExpectedDiff
{
    public string FindingId { get; init; } = string.Empty;
    public string Status { get; init; } = "Pass";
}

internal sealed record ScenarioBaseline
{
    public string FindingId { get; init; } = string.Empty;
    public string Status { get; init; } = "Pass";
    public string? RuleName { get; init; }
    public string? RuleAction { get; init; }
    public string? Notes { get; init; }
    public double Score { get; init; }
    public string? ConfigVersion { get; init; }
    public Dictionary<string, double>? Inputs { get; init; }
}

internal sealed record ScenarioResult(string ScenarioName)
{
    public bool Success { get; init; } = true;
    public int ChangedCount { get; init; }
    public List<string> Failures { get; } = new();
    public Dictionary<string, string> ActualStatuses { get; } = new(StringComparer.OrdinalIgnoreCase);
}

internal sealed class NullPolicySnapshotRepository : IPolicySnapshotRepository
{
    public Task AddAsync(PolicySnapshot snapshot, CancellationToken cancellationToken = default) => Task.CompletedTask;

    public Task<PolicySnapshot?> GetLatestAsync(CancellationToken cancellationToken = default) => Task.FromResult<PolicySnapshot?>(null);

    public Task<IReadOnlyList<PolicySnapshot>> ListAsync(int limit, CancellationToken cancellationToken = default)
        => Task.FromResult<IReadOnlyList<PolicySnapshot>>(Array.Empty<PolicySnapshot>());
}

internal sealed class NullPolicyAuditRepository : IPolicyAuditRepository
{
    public Task AddAsync(PolicyAuditEntry entry, CancellationToken cancellationToken = default) => Task.CompletedTask;

    public Task<IReadOnlyList<PolicyAuditEntry>> ListAsync(int limit, CancellationToken cancellationToken = default)
        => Task.FromResult<IReadOnlyList<PolicyAuditEntry>>(Array.Empty<PolicyAuditEntry>());
}