git.stella-ops.org/src/Scanner/StellaOps.Scanner.WebService/Services/RuntimePolicyService.cs

using System;
using System.Collections.Immutable;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Globalization;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Policy;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.WebService.Contracts;
using StellaOps.Scanner.WebService.Options;
using StellaOps.Zastava.Core.Contracts;
using RuntimePolicyVerdict = StellaOps.Zastava.Core.Contracts.PolicyVerdict;
using CanonicalPolicyVerdict = StellaOps.Policy.PolicyVerdict;
using CanonicalPolicyVerdictStatus = StellaOps.Policy.PolicyVerdictStatus;

namespace StellaOps.Scanner.WebService.Services;

internal interface IRuntimePolicyService
{
    Task<RuntimePolicyEvaluationResult> EvaluateAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken);
}

internal sealed class RuntimePolicyService : IRuntimePolicyService
{
    private const int MaxBuildIdsPerImage = 3;

    private static readonly Meter PolicyMeter = new("StellaOps.Scanner.RuntimePolicy", "1.0.0");
    private static readonly Counter<long> PolicyEvaluations = PolicyMeter.CreateCounter<long>("scanner.runtime.policy.requests", unit: "1", description: "Total runtime policy evaluation requests processed.");
    private static readonly Histogram<double> PolicyEvaluationLatencyMs = PolicyMeter.CreateHistogram<double>("scanner.runtime.policy.latency.ms", unit: "ms", description: "Latency for runtime policy evaluations.");

    private readonly LinkRepository _linkRepository;
    private readonly ArtifactRepository _artifactRepository;
    private readonly RuntimeEventRepository _runtimeEventRepository;
    private readonly PolicySnapshotStore _policySnapshotStore;
    private readonly PolicyPreviewService _policyPreviewService;
    private readonly ILinksetResolver _linksetResolver;
    private readonly IOptionsMonitor<ScannerWebServiceOptions> _optionsMonitor;
    private readonly TimeProvider _timeProvider;
    private readonly IRuntimeAttestationVerifier _attestationVerifier;
    private readonly ILogger<RuntimePolicyService> _logger;

    public RuntimePolicyService(
        LinkRepository linkRepository,
        ArtifactRepository artifactRepository,
        RuntimeEventRepository runtimeEventRepository,
        PolicySnapshotStore policySnapshotStore,
        PolicyPreviewService policyPreviewService,
        ILinksetResolver linksetResolver,
        IOptionsMonitor<ScannerWebServiceOptions> optionsMonitor,
        TimeProvider timeProvider,
        IRuntimeAttestationVerifier attestationVerifier,
        ILogger<RuntimePolicyService> logger)
    {
        _linkRepository = linkRepository ?? throw new ArgumentNullException(nameof(linkRepository));
        _artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
        _runtimeEventRepository = runtimeEventRepository ?? throw new ArgumentNullException(nameof(runtimeEventRepository));
        _policySnapshotStore = policySnapshotStore ?? throw new ArgumentNullException(nameof(policySnapshotStore));
        _policyPreviewService = policyPreviewService ?? throw new ArgumentNullException(nameof(policyPreviewService));
        _linksetResolver = linksetResolver ?? throw new ArgumentNullException(nameof(linksetResolver));
        _optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
        _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
        _attestationVerifier = attestationVerifier ?? throw new ArgumentNullException(nameof(attestationVerifier));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
    }

    public async Task<RuntimePolicyEvaluationResult> EvaluateAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken)
    {
        ArgumentNullException.ThrowIfNull(request);

        var runtimeOptions = _optionsMonitor.CurrentValue.Runtime ?? new ScannerWebServiceOptions.RuntimeOptions();
        var ttlSeconds = Math.Max(1, runtimeOptions.PolicyCacheTtlSeconds);

        var now = _timeProvider.GetUtcNow();
        var expiresAt = now.AddSeconds(ttlSeconds);

        var stopwatch = Stopwatch.StartNew();
        var snapshot = await _policySnapshotStore.GetLatestAsync(cancellationToken).ConfigureAwait(false);

        var determinism = _optionsMonitor.CurrentValue.Determinism ?? new ScannerWebServiceOptions.DeterminismOptions();
        if (!string.IsNullOrWhiteSpace(determinism.PolicySnapshotId))
        {
            if (snapshot is null || !string.Equals(snapshot.RevisionId, determinism.PolicySnapshotId, StringComparison.Ordinal))
            {
                throw new InvalidOperationException($"Deterministic policy pin {determinism.PolicySnapshotId} is not present; current revision is {snapshot?.RevisionId ?? "none"}.");
            }
        }

        var policyRevision = snapshot?.RevisionId;
        var policyDigest = snapshot?.Digest;

        var results = new Dictionary<string, RuntimePolicyImageDecision>(StringComparer.Ordinal);
        var evaluationTags = new KeyValuePair<string, object?>[]
        {
            new("policy_revision", policyRevision ?? "none"),
            new("namespace", request.Namespace ?? "unspecified")
        };

        var buildIdObservations = await _runtimeEventRepository
            .GetRecentBuildIdsAsync(request.Images, MaxBuildIdsPerImage, cancellationToken)
            .ConfigureAwait(false);

        try
        {
            var evaluated = new HashSet<string>(StringComparer.Ordinal);
            foreach (var image in request.Images)
            {
                if (!evaluated.Add(image))
                {
                    continue;
                }

                var metadata = await ResolveImageMetadataAsync(image, cancellationToken).ConfigureAwait(false);
                var (findings, heuristicReasons) = BuildFindings(image, metadata, request.Namespace);
                if (snapshot is null)
                {
                    heuristicReasons.Add("policy.snapshot.missing");
                }

                ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts = ImmutableArray<CanonicalPolicyVerdict>.Empty;
                ImmutableArray<PolicyIssue> issues = ImmutableArray<PolicyIssue>.Empty;
                IReadOnlyList<LinksetSummaryDto> linksets = Array.Empty<LinksetSummaryDto>();

                try
                {
                    if (!findings.IsDefaultOrEmpty && findings.Length > 0)
                    {
                        var previewRequest = new PolicyPreviewRequest(
                            image,
                            findings,
                            ImmutableArray<CanonicalPolicyVerdict>.Empty,
                            snapshot,
                            ProposedPolicy: null);

                        var preview = await _policyPreviewService.PreviewAsync(previewRequest, cancellationToken).ConfigureAwait(false);
                        issues = preview.Issues;
                        if (!preview.Diffs.IsDefaultOrEmpty)
                        {
                            projectedVerdicts = preview.Diffs.Select(diff => diff.Projected).ToImmutableArray();
                            linksets = await _linksetResolver.ResolveAsync(projectedVerdicts, cancellationToken).ConfigureAwait(false);
                        }
                    }
                }
                catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
                {
                    _logger.LogWarning(ex, "Runtime policy preview failed for image {ImageDigest}; falling back to heuristic evaluation.", image);
                }

                var normalizedImage = image.Trim().ToLowerInvariant();
                buildIdObservations.TryGetValue(normalizedImage, out var buildIdObservation);

                var decision = await BuildDecisionAsync(
                    image,
                    metadata,
                    heuristicReasons,
                    projectedVerdicts,
                    issues,
                    policyDigest,
                    linksets,
                    buildIdObservation?.BuildIds,
                    cancellationToken).ConfigureAwait(false);

                results[image] = decision;

                _logger.LogInformation("Runtime policy evaluated image {ImageDigest} with verdict {Verdict} (Signed: {Signed}, HasSbom: {HasSbom}, Reasons: {ReasonsCount})",
                    image,
                    decision.PolicyVerdict,
                    decision.Signed,
                    decision.HasSbomReferrers,
                    decision.Reasons.Count);
            }
        }
        finally
        {
            stopwatch.Stop();
            PolicyEvaluationLatencyMs.Record(stopwatch.Elapsed.TotalMilliseconds, evaluationTags);
        }

        PolicyEvaluations.Add(results.Count, evaluationTags);

        var evaluationResult = new RuntimePolicyEvaluationResult(
            ttlSeconds,
            expiresAt,
            policyRevision,
            new ReadOnlyDictionary<string, RuntimePolicyImageDecision>(results));

        return evaluationResult;
    }

    private async Task<RuntimeImageMetadata> ResolveImageMetadataAsync(string imageDigest, CancellationToken cancellationToken)
    {
        var links = await _linkRepository.ListBySourceAsync(LinkSourceType.Image, imageDigest, cancellationToken).ConfigureAwait(false);
        if (links.Count == 0)
        {
            return new RuntimeImageMetadata(imageDigest, false, false, null, MissingMetadata: true);
        }

        var hasSbom = false;
        var signed = false;
        RuntimePolicyRekorReference? rekor = null;

        foreach (var link in links)
        {
            var artifact = await _artifactRepository.GetAsync(link.ArtifactId, cancellationToken).ConfigureAwait(false);
            if (artifact is null)
            {
                continue;
            }

            switch (artifact.Type)
            {
                case ArtifactDocumentType.ImageBom:
                    hasSbom = true;
                    break;
                case ArtifactDocumentType.Attestation:
                    signed = true;
                    if (artifact.Rekor is { } rekorReference)
                    {
                        rekor = new RuntimePolicyRekorReference(
                            Normalize(rekorReference.Uuid),
                            Normalize(rekorReference.Url),
                            rekorReference.Index.HasValue);
                    }
                    break;
            }
        }

        return new RuntimeImageMetadata(imageDigest, signed, hasSbom, rekor, MissingMetadata: false);
    }

    private (ImmutableArray<PolicyFinding> Findings, List<string> HeuristicReasons) BuildFindings(string imageDigest, RuntimeImageMetadata metadata, string? @namespace)
    {
        var findings = ImmutableArray.CreateBuilder<PolicyFinding>();
        var heuristics = new List<string>();

        findings.Add(PolicyFinding.Create(
            $"{imageDigest}#baseline",
            PolicySeverity.None,
            environment: @namespace,
            source: "scanner.runtime"));

        if (metadata.MissingMetadata)
        {
            const string reason = "image.metadata.missing";
            heuristics.Add(reason);
            findings.Add(PolicyFinding.Create(
                $"{imageDigest}#metadata",
                PolicySeverity.Critical,
                environment: @namespace,
                source: "scanner.runtime",
                tags: ImmutableArray.Create(reason)));
        }

        if (!metadata.Signed)
        {
            const string reason = "unsigned";
            heuristics.Add(reason);
            findings.Add(PolicyFinding.Create(
                $"{imageDigest}#signature",
                PolicySeverity.High,
                environment: @namespace,
                source: "scanner.runtime",
                tags: ImmutableArray.Create(reason)));
        }

        if (!metadata.HasSbomReferrers)
        {
            const string reason = "missing SBOM";
            heuristics.Add(reason);
            findings.Add(PolicyFinding.Create(
                $"{imageDigest}#sbom",
                PolicySeverity.High,
                environment: @namespace,
                source: "scanner.runtime",
                tags: ImmutableArray.Create(reason)));
        }

        return (findings.ToImmutable(), heuristics);
    }

    private async Task<RuntimePolicyImageDecision> BuildDecisionAsync(
        string imageDigest,
        RuntimeImageMetadata metadata,
        List<string> heuristicReasons,
        ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts,
        ImmutableArray<PolicyIssue> issues,
        string? policyDigest,
        IReadOnlyList<LinksetSummaryDto> linksets,
        IReadOnlyList<string>? buildIds,
        CancellationToken cancellationToken)
    {
        var reasons = new List<string>(heuristicReasons);

        var overallVerdict = MapVerdict(projectedVerdicts, heuristicReasons);

        if (!projectedVerdicts.IsDefaultOrEmpty)
        {
            foreach (var verdict in projectedVerdicts)
            {
                if (verdict.Status == CanonicalPolicyVerdictStatus.Pass)
                {
                    continue;
                }

                if (!string.IsNullOrWhiteSpace(verdict.RuleName))
                {
                    reasons.Add($"policy.rule.{verdict.RuleName}");
                }
                else
                {
                    reasons.Add($"policy.status.{verdict.Status.ToString().ToLowerInvariant()}");
                }
            }
        }

        var confidence = ComputeConfidence(projectedVerdicts, overallVerdict);
        var quieted = !projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Any(v => v.Quiet);
        var quietedBy = !projectedVerdicts.IsDefaultOrEmpty
            ? projectedVerdicts.FirstOrDefault(v => !string.IsNullOrWhiteSpace(v.QuietedBy))?.QuietedBy
            : null;

        var metadataPayload = BuildMetadataPayload(heuristicReasons, projectedVerdicts, issues, policyDigest);

        var rekor = metadata.Rekor;
        var verified = await _attestationVerifier.VerifyAsync(imageDigest, metadata.Rekor, cancellationToken).ConfigureAwait(false);
        if (rekor is not null && verified.HasValue)
        {
            rekor = rekor with { Verified = verified.Value };
        }

        var normalizedReasons = reasons
            .Where(reason => !string.IsNullOrWhiteSpace(reason))
            .Distinct(StringComparer.Ordinal)
            .ToArray();

        return new RuntimePolicyImageDecision(
            overallVerdict,
            metadata.Signed,
            metadata.HasSbomReferrers,
            normalizedReasons,
            rekor,
            metadataPayload,
            confidence,
            quieted,
            quietedBy,
            buildIds,
            linksets);
    }

    private RuntimePolicyVerdict MapVerdict(ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts, IReadOnlyList<string> heuristicReasons)
    {
        if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0)
        {
            var statuses = projectedVerdicts.Select(v => v.Status).ToArray();
            if (statuses.Any(status => status == CanonicalPolicyVerdictStatus.Blocked))
            {
                return RuntimePolicyVerdict.Fail;
            }

            if (statuses.Any(status =>
                    status is CanonicalPolicyVerdictStatus.Warned
                        or CanonicalPolicyVerdictStatus.Deferred
                        or CanonicalPolicyVerdictStatus.Escalated
                        or CanonicalPolicyVerdictStatus.RequiresVex))
            {
                return RuntimePolicyVerdict.Warn;
            }

            return RuntimePolicyVerdict.Pass;
        }

        if (heuristicReasons.Contains("image.metadata.missing", StringComparer.Ordinal) ||
            heuristicReasons.Contains("unsigned", StringComparer.Ordinal) ||
            heuristicReasons.Contains("missing SBOM", StringComparer.Ordinal))
        {
            return RuntimePolicyVerdict.Fail;
        }

        if (heuristicReasons.Contains("policy.snapshot.missing", StringComparer.Ordinal))
        {
            return RuntimePolicyVerdict.Warn;
        }

        return RuntimePolicyVerdict.Pass;
    }

    private IDictionary<string, object?>? BuildMetadataPayload(
        IReadOnlyList<string> heuristics,
        ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts,
        ImmutableArray<PolicyIssue> issues,
        string? policyDigest)
    {
        var payload = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
        {
            ["heuristics"] = heuristics,
            ["evaluatedAt"] = _timeProvider.GetUtcNow().UtcDateTime
        };

        if (!string.IsNullOrWhiteSpace(policyDigest))
        {
            payload["policyDigest"] = policyDigest;
        }

        if (!issues.IsDefaultOrEmpty && issues.Length > 0)
        {
            payload["issues"] = issues.Select(issue => new
            {
                code = issue.Code,
                severity = issue.Severity.ToString(),
                message = issue.Message,
                path = issue.Path
            }).ToArray();
        }

        if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0)
        {
            payload["findings"] = projectedVerdicts.Select(verdict => new
            {
                id = verdict.FindingId,
                status = verdict.Status.ToString().ToLowerInvariant(),
                rule = verdict.RuleName,
                action = verdict.RuleAction,
                score = verdict.Score,
                quiet = verdict.Quiet,
                quietedBy = verdict.QuietedBy,
                inputs = verdict.GetInputs(),
                confidence = verdict.UnknownConfidence,
                confidenceBand = verdict.ConfidenceBand,
                sourceTrust = verdict.SourceTrust,
                reachability = verdict.Reachability
            }).ToArray();
        }

        return payload.Count == 0 ? null : payload;
    }

    private static double ComputeConfidence(ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts, RuntimePolicyVerdict overall)
    {
        if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0)
        {
            var confidences = projectedVerdicts
                .Select(v => v.UnknownConfidence)
                .Where(value => value.HasValue)
                .Select(value => value!.Value)
                .ToArray();

            if (confidences.Length > 0)
            {
                return Math.Clamp(confidences.Average(), 0.0, 1.0);
            }
        }

        return overall switch
        {
            RuntimePolicyVerdict.Pass => 0.95,
            RuntimePolicyVerdict.Warn => 0.5,
            RuntimePolicyVerdict.Fail => 0.1,
            _ => 0.25
        };
    }

    private static string? Normalize(string? value)
        => string.IsNullOrWhiteSpace(value) ? null : value;
}

internal interface IRuntimeAttestationVerifier
{
    ValueTask<bool?> VerifyAsync(string imageDigest, RuntimePolicyRekorReference? rekor, CancellationToken cancellationToken);
}

internal sealed class RuntimeAttestationVerifier : IRuntimeAttestationVerifier
{
    private readonly ILogger<RuntimeAttestationVerifier> _logger;

    public RuntimeAttestationVerifier(ILogger<RuntimeAttestationVerifier> logger)
    {
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
    }

    public ValueTask<bool?> VerifyAsync(string imageDigest, RuntimePolicyRekorReference? rekor, CancellationToken cancellationToken)
    {
        if (rekor is null)
        {
            return ValueTask.FromResult<bool?>(null);
        }

        if (rekor.Verified.HasValue)
        {
            return ValueTask.FromResult(rekor.Verified);
        }

        _logger.LogDebug("No attestation verification metadata available for image {ImageDigest}.", imageDigest);
        return ValueTask.FromResult<bool?>(null);
    }
}

internal sealed record RuntimePolicyEvaluationRequest(
    string? Namespace,
    IReadOnlyDictionary<string, string> Labels,
    IReadOnlyList<string> Images);

internal sealed record RuntimePolicyEvaluationResult(
    int TtlSeconds,
    DateTimeOffset ExpiresAtUtc,
    string? PolicyRevision,
    IReadOnlyDictionary<string, RuntimePolicyImageDecision> Results);

internal sealed record RuntimePolicyImageDecision(
    RuntimePolicyVerdict PolicyVerdict,
    bool Signed,
    bool HasSbomReferrers,
    IReadOnlyList<string> Reasons,
    RuntimePolicyRekorReference? Rekor,
    IDictionary<string, object?>? Metadata,
    double Confidence,
    bool Quieted,
    string? QuietedBy,
    IReadOnlyList<string>? BuildIds,
    IReadOnlyList<LinksetSummaryDto> Linksets);

internal sealed record RuntimePolicyRekorReference(string? Uuid, string? Url, bool? Verified);

internal sealed record RuntimeImageMetadata(
    string ImageDigest,
    bool Signed,
    bool HasSbomReferrers,
    RuntimePolicyRekorReference? Rekor,
    bool MissingMetadata);