using System; using System.Collections.Immutable; using System.Collections.ObjectModel; using System.Diagnostics; using System.Diagnostics.Metrics; using System.Linq; using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Policy; using StellaOps.Scanner.Storage.Catalog; using StellaOps.Scanner.Storage.Repositories; using StellaOps.Scanner.WebService.Contracts; using StellaOps.Scanner.WebService.Options; using StellaOps.Zastava.Core.Contracts; using RuntimePolicyVerdict = StellaOps.Zastava.Core.Contracts.PolicyVerdict; using CanonicalPolicyVerdict = StellaOps.Policy.PolicyVerdict; using CanonicalPolicyVerdictStatus = StellaOps.Policy.PolicyVerdictStatus; namespace StellaOps.Scanner.WebService.Services; internal interface IRuntimePolicyService { Task EvaluateAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken); } internal sealed class RuntimePolicyService : IRuntimePolicyService { private const int MaxBuildIdsPerImage = 3; private static readonly Meter PolicyMeter = new("StellaOps.Scanner.RuntimePolicy", "1.0.0"); private static readonly Counter PolicyEvaluations = PolicyMeter.CreateCounter("scanner.runtime.policy.requests", unit: "1", description: "Total runtime policy evaluation requests processed."); private static readonly Histogram PolicyEvaluationLatencyMs = PolicyMeter.CreateHistogram("scanner.runtime.policy.latency.ms", unit: "ms", description: "Latency for runtime policy evaluations."); private readonly LinkRepository _linkRepository; private readonly ArtifactRepository _artifactRepository; private readonly RuntimeEventRepository _runtimeEventRepository; private readonly PolicySnapshotStore _policySnapshotStore; private readonly PolicyPreviewService _policyPreviewService; private readonly ILinksetResolver _linksetResolver; private readonly IOptionsMonitor _optionsMonitor; private readonly TimeProvider _timeProvider; private readonly IRuntimeAttestationVerifier _attestationVerifier; private readonly ILogger _logger; public RuntimePolicyService( LinkRepository linkRepository, ArtifactRepository artifactRepository, RuntimeEventRepository runtimeEventRepository, PolicySnapshotStore policySnapshotStore, PolicyPreviewService policyPreviewService, ILinksetResolver linksetResolver, IOptionsMonitor optionsMonitor, TimeProvider timeProvider, IRuntimeAttestationVerifier attestationVerifier, ILogger logger) { _linkRepository = linkRepository ?? throw new ArgumentNullException(nameof(linkRepository)); _artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository)); _runtimeEventRepository = runtimeEventRepository ?? throw new ArgumentNullException(nameof(runtimeEventRepository)); _policySnapshotStore = policySnapshotStore ?? throw new ArgumentNullException(nameof(policySnapshotStore)); _policyPreviewService = policyPreviewService ?? throw new ArgumentNullException(nameof(policyPreviewService)); _linksetResolver = linksetResolver ?? throw new ArgumentNullException(nameof(linksetResolver)); _optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); _attestationVerifier = attestationVerifier ?? throw new ArgumentNullException(nameof(attestationVerifier)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public async Task EvaluateAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(request); var runtimeOptions = _optionsMonitor.CurrentValue.Runtime ?? new ScannerWebServiceOptions.RuntimeOptions(); var ttlSeconds = Math.Max(1, runtimeOptions.PolicyCacheTtlSeconds); var now = _timeProvider.GetUtcNow(); var expiresAt = now.AddSeconds(ttlSeconds); var stopwatch = Stopwatch.StartNew(); var snapshot = await _policySnapshotStore.GetLatestAsync(cancellationToken).ConfigureAwait(false); var determinism = _optionsMonitor.CurrentValue.Determinism ?? new ScannerWebServiceOptions.DeterminismOptions(); if (!string.IsNullOrWhiteSpace(determinism.PolicySnapshotId)) { if (snapshot is null || !string.Equals(snapshot.RevisionId, determinism.PolicySnapshotId, StringComparison.Ordinal)) { throw new InvalidOperationException($"Deterministic policy pin {determinism.PolicySnapshotId} is not present; current revision is {snapshot?.RevisionId ?? "none"}."); } } var policyRevision = snapshot?.RevisionId; var policyDigest = snapshot?.Digest; var results = new Dictionary(StringComparer.Ordinal); var evaluationTags = new KeyValuePair[] { new("policy_revision", policyRevision ?? "none"), new("namespace", request.Namespace ?? "unspecified") }; var buildIdObservations = await _runtimeEventRepository .GetRecentBuildIdsAsync(request.Images, MaxBuildIdsPerImage, cancellationToken) .ConfigureAwait(false); try { var evaluated = new HashSet(StringComparer.Ordinal); foreach (var image in request.Images) { if (!evaluated.Add(image)) { continue; } var metadata = await ResolveImageMetadataAsync(image, cancellationToken).ConfigureAwait(false); var (findings, heuristicReasons) = BuildFindings(image, metadata, request.Namespace); if (snapshot is null) { heuristicReasons.Add("policy.snapshot.missing"); } ImmutableArray projectedVerdicts = ImmutableArray.Empty; ImmutableArray issues = ImmutableArray.Empty; IReadOnlyList linksets = Array.Empty(); try { if (!findings.IsDefaultOrEmpty && findings.Length > 0) { var previewRequest = new PolicyPreviewRequest( image, findings, ImmutableArray.Empty, snapshot, ProposedPolicy: null); var preview = await _policyPreviewService.PreviewAsync(previewRequest, cancellationToken).ConfigureAwait(false); issues = preview.Issues; if (!preview.Diffs.IsDefaultOrEmpty) { projectedVerdicts = preview.Diffs.Select(diff => diff.Projected).ToImmutableArray(); linksets = await _linksetResolver.ResolveAsync(projectedVerdicts, cancellationToken).ConfigureAwait(false); } } } catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { _logger.LogWarning(ex, "Runtime policy preview failed for image {ImageDigest}; falling back to heuristic evaluation.", image); } var normalizedImage = image.Trim().ToLowerInvariant(); buildIdObservations.TryGetValue(normalizedImage, out var buildIdObservation); var decision = await BuildDecisionAsync( image, metadata, heuristicReasons, projectedVerdicts, issues, policyDigest, linksets, buildIdObservation?.BuildIds, cancellationToken).ConfigureAwait(false); results[image] = decision; _logger.LogInformation("Runtime policy evaluated image {ImageDigest} with verdict {Verdict} (Signed: {Signed}, HasSbom: {HasSbom}, Reasons: {ReasonsCount})", image, decision.PolicyVerdict, decision.Signed, decision.HasSbomReferrers, decision.Reasons.Count); } } finally { stopwatch.Stop(); PolicyEvaluationLatencyMs.Record(stopwatch.Elapsed.TotalMilliseconds, evaluationTags); } PolicyEvaluations.Add(results.Count, evaluationTags); var evaluationResult = new RuntimePolicyEvaluationResult( ttlSeconds, expiresAt, policyRevision, new ReadOnlyDictionary(results)); return evaluationResult; } private async Task ResolveImageMetadataAsync(string imageDigest, CancellationToken cancellationToken) { var links = await _linkRepository.ListBySourceAsync(LinkSourceType.Image, imageDigest, cancellationToken).ConfigureAwait(false); if (links.Count == 0) { return new RuntimeImageMetadata(imageDigest, false, false, null, MissingMetadata: true); } var hasSbom = false; var signed = false; RuntimePolicyRekorReference? rekor = null; foreach (var link in links) { var artifact = await _artifactRepository.GetAsync(link.ArtifactId, cancellationToken).ConfigureAwait(false); if (artifact is null) { continue; } switch (artifact.Type) { case ArtifactDocumentType.ImageBom: hasSbom = true; break; case ArtifactDocumentType.Attestation: signed = true; if (artifact.Rekor is { } rekorReference) { rekor = new RuntimePolicyRekorReference( Normalize(rekorReference.Uuid), Normalize(rekorReference.Url), rekorReference.Index.HasValue); } break; } } return new RuntimeImageMetadata(imageDigest, signed, hasSbom, rekor, MissingMetadata: false); } private (ImmutableArray Findings, List HeuristicReasons) BuildFindings(string imageDigest, RuntimeImageMetadata metadata, string? @namespace) { var findings = ImmutableArray.CreateBuilder(); var heuristics = new List(); findings.Add(PolicyFinding.Create( $"{imageDigest}#baseline", PolicySeverity.None, environment: @namespace, source: "scanner.runtime")); if (metadata.MissingMetadata) { const string reason = "image.metadata.missing"; heuristics.Add(reason); findings.Add(PolicyFinding.Create( $"{imageDigest}#metadata", PolicySeverity.Critical, environment: @namespace, source: "scanner.runtime", tags: ImmutableArray.Create(reason))); } if (!metadata.Signed) { const string reason = "unsigned"; heuristics.Add(reason); findings.Add(PolicyFinding.Create( $"{imageDigest}#signature", PolicySeverity.High, environment: @namespace, source: "scanner.runtime", tags: ImmutableArray.Create(reason))); } if (!metadata.HasSbomReferrers) { const string reason = "missing SBOM"; heuristics.Add(reason); findings.Add(PolicyFinding.Create( $"{imageDigest}#sbom", PolicySeverity.High, environment: @namespace, source: "scanner.runtime", tags: ImmutableArray.Create(reason))); } return (findings.ToImmutable(), heuristics); } private async Task BuildDecisionAsync( string imageDigest, RuntimeImageMetadata metadata, List heuristicReasons, ImmutableArray projectedVerdicts, ImmutableArray issues, string? policyDigest, IReadOnlyList linksets, IReadOnlyList? buildIds, CancellationToken cancellationToken) { var reasons = new List(heuristicReasons); var overallVerdict = MapVerdict(projectedVerdicts, heuristicReasons); if (!projectedVerdicts.IsDefaultOrEmpty) { foreach (var verdict in projectedVerdicts) { if (verdict.Status == CanonicalPolicyVerdictStatus.Pass) { continue; } if (!string.IsNullOrWhiteSpace(verdict.RuleName)) { reasons.Add($"policy.rule.{verdict.RuleName}"); } else { reasons.Add($"policy.status.{verdict.Status.ToString().ToLowerInvariant()}"); } } } var confidence = ComputeConfidence(projectedVerdicts, overallVerdict); var quieted = !projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Any(v => v.Quiet); var quietedBy = !projectedVerdicts.IsDefaultOrEmpty ? projectedVerdicts.FirstOrDefault(v => !string.IsNullOrWhiteSpace(v.QuietedBy))?.QuietedBy : null; var metadataPayload = BuildMetadataPayload(heuristicReasons, projectedVerdicts, issues, policyDigest); var rekor = metadata.Rekor; var verified = await _attestationVerifier.VerifyAsync(imageDigest, metadata.Rekor, cancellationToken).ConfigureAwait(false); if (rekor is not null && verified.HasValue) { rekor = rekor with { Verified = verified.Value }; } var normalizedReasons = reasons .Where(reason => !string.IsNullOrWhiteSpace(reason)) .Distinct(StringComparer.Ordinal) .ToArray(); return new RuntimePolicyImageDecision( overallVerdict, metadata.Signed, metadata.HasSbomReferrers, normalizedReasons, rekor, metadataPayload, confidence, quieted, quietedBy, buildIds, linksets); } private RuntimePolicyVerdict MapVerdict(ImmutableArray projectedVerdicts, IReadOnlyList heuristicReasons) { if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0) { var statuses = projectedVerdicts.Select(v => v.Status).ToArray(); if (statuses.Any(status => status == CanonicalPolicyVerdictStatus.Blocked)) { return RuntimePolicyVerdict.Fail; } if (statuses.Any(status => status is CanonicalPolicyVerdictStatus.Warned or CanonicalPolicyVerdictStatus.Deferred or CanonicalPolicyVerdictStatus.Escalated or CanonicalPolicyVerdictStatus.RequiresVex)) { return RuntimePolicyVerdict.Warn; } return RuntimePolicyVerdict.Pass; } if (heuristicReasons.Contains("image.metadata.missing", StringComparer.Ordinal) || heuristicReasons.Contains("unsigned", StringComparer.Ordinal) || heuristicReasons.Contains("missing SBOM", StringComparer.Ordinal)) { return RuntimePolicyVerdict.Fail; } if (heuristicReasons.Contains("policy.snapshot.missing", StringComparer.Ordinal)) { return RuntimePolicyVerdict.Warn; } return RuntimePolicyVerdict.Pass; } private IDictionary? BuildMetadataPayload( IReadOnlyList heuristics, ImmutableArray projectedVerdicts, ImmutableArray issues, string? policyDigest) { var payload = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["heuristics"] = heuristics, ["evaluatedAt"] = _timeProvider.GetUtcNow().UtcDateTime }; if (!string.IsNullOrWhiteSpace(policyDigest)) { payload["policyDigest"] = policyDigest; } if (!issues.IsDefaultOrEmpty && issues.Length > 0) { payload["issues"] = issues.Select(issue => new { code = issue.Code, severity = issue.Severity.ToString(), message = issue.Message, path = issue.Path }).ToArray(); } if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0) { payload["findings"] = projectedVerdicts.Select(verdict => new { id = verdict.FindingId, status = verdict.Status.ToString().ToLowerInvariant(), rule = verdict.RuleName, action = verdict.RuleAction, score = verdict.Score, quiet = verdict.Quiet, quietedBy = verdict.QuietedBy, inputs = verdict.GetInputs(), confidence = verdict.UnknownConfidence, confidenceBand = verdict.ConfidenceBand, sourceTrust = verdict.SourceTrust, reachability = verdict.Reachability }).ToArray(); } return payload.Count == 0 ? null : payload; } private static double ComputeConfidence(ImmutableArray projectedVerdicts, RuntimePolicyVerdict overall) { if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0) { var confidences = projectedVerdicts .Select(v => v.UnknownConfidence) .Where(value => value.HasValue) .Select(value => value!.Value) .ToArray(); if (confidences.Length > 0) { return Math.Clamp(confidences.Average(), 0.0, 1.0); } } return overall switch { RuntimePolicyVerdict.Pass => 0.95, RuntimePolicyVerdict.Warn => 0.5, RuntimePolicyVerdict.Fail => 0.1, _ => 0.25 }; } private static string? Normalize(string? value) => string.IsNullOrWhiteSpace(value) ? null : value; } internal interface IRuntimeAttestationVerifier { ValueTask VerifyAsync(string imageDigest, RuntimePolicyRekorReference? rekor, CancellationToken cancellationToken); } internal sealed class RuntimeAttestationVerifier : IRuntimeAttestationVerifier { private readonly ILogger _logger; public RuntimeAttestationVerifier(ILogger logger) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public ValueTask VerifyAsync(string imageDigest, RuntimePolicyRekorReference? rekor, CancellationToken cancellationToken) { if (rekor is null) { return ValueTask.FromResult(null); } if (rekor.Verified.HasValue) { return ValueTask.FromResult(rekor.Verified); } _logger.LogDebug("No attestation verification metadata available for image {ImageDigest}.", imageDigest); return ValueTask.FromResult(null); } } internal sealed record RuntimePolicyEvaluationRequest( string? Namespace, IReadOnlyDictionary Labels, IReadOnlyList Images); internal sealed record RuntimePolicyEvaluationResult( int TtlSeconds, DateTimeOffset ExpiresAtUtc, string? PolicyRevision, IReadOnlyDictionary Results); internal sealed record RuntimePolicyImageDecision( RuntimePolicyVerdict PolicyVerdict, bool Signed, bool HasSbomReferrers, IReadOnlyList Reasons, RuntimePolicyRekorReference? Rekor, IDictionary? Metadata, double Confidence, bool Quieted, string? QuietedBy, IReadOnlyList? BuildIds, IReadOnlyList Linksets); internal sealed record RuntimePolicyRekorReference(string? Uuid, string? Url, bool? Verified); internal sealed record RuntimeImageMetadata( string ImageDigest, bool Signed, bool HasSbomReferrers, RuntimePolicyRekorReference? Rekor, bool MissingMetadata);