up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

This commit is contained in:
Vladimir Moushkov
2025-10-24 19:19:23 +03:00
parent 17d861e4ab
commit b51037a9b8
72 changed files with 6070 additions and 151 deletions

View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
@@ -210,6 +211,71 @@ rules:
Assert.NotNull(decision.Rekor);
Assert.Equal("rekor-uuid", decision.Rekor!.Uuid);
Assert.True(decision.Rekor.Verified);
Assert.NotNull(decision.Confidence);
Assert.InRange(decision.Confidence!.Value, 0.0, 1.0);
Assert.False(decision.Quieted.GetValueOrDefault());
Assert.Null(decision.QuietedBy);
var metadataString = decision.Metadata;
Console.WriteLine($"Runtime policy metadata: {metadataString ?? "<null>"}");
Assert.False(string.IsNullOrWhiteSpace(metadataString));
using var metadataDocument = JsonDocument.Parse(decision.Metadata!);
Assert.True(metadataDocument.RootElement.TryGetProperty("heuristics", out _));
}
[Fact]
public async Task RuntimePolicyEndpointFlagsUnsignedAndMissingSbom()
{
using var factory = new ScannerApplicationFactory();
using var client = factory.CreateClient();
const string imageDigest = "sha256:feedface";
using (var scope = factory.Services.CreateScope())
{
var collections = scope.ServiceProvider.GetRequiredService<MongoCollectionProvider>();
var policyStore = scope.ServiceProvider.GetRequiredService<PolicySnapshotStore>();
const string policyYaml = """
version: "1.0"
rules: []
""";
await policyStore.SaveAsync(
new PolicySnapshotContent(policyYaml, PolicyDocumentFormat.Yaml, "tester", "tests", "baseline"),
CancellationToken.None);
// Intentionally skip artifacts/links to simulate missing metadata.
await collections.RuntimeEvents.DeleteManyAsync(Builders<RuntimeEventDocument>.Filter.Empty);
}
var response = await client.PostAsJsonAsync("/api/v1/policy/runtime", new RuntimePolicyRequestDto
{
Namespace = "payments",
Images = new[] { imageDigest }
});
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var payload = await response.Content.ReadFromJsonAsync<RuntimePolicyResponseDto>();
Assert.NotNull(payload);
var decision = payload!.Results[imageDigest];
Assert.Equal("fail", decision.PolicyVerdict);
Assert.False(decision.Signed);
Assert.False(decision.HasSbomReferrers);
Assert.Contains("image.metadata.missing", decision.Reasons);
Assert.Contains("unsigned", decision.Reasons);
Assert.Contains("missing SBOM", decision.Reasons);
Assert.NotNull(decision.Confidence);
Assert.InRange(decision.Confidence!.Value, 0.0, 1.0);
if (!string.IsNullOrWhiteSpace(decision.Metadata))
{
using var failureMetadata = JsonDocument.Parse(decision.Metadata!);
if (failureMetadata.RootElement.TryGetProperty("heuristics", out var heuristicsElement))
{
var heuristics = heuristicsElement.EnumerateArray().Select(item => item.GetString()).ToArray();
Assert.Contains("image.metadata.missing", heuristics);
Assert.Contains("unsigned", heuristics);
}
}
}
[Fact]

View File

@@ -54,9 +54,21 @@ public sealed record RuntimePolicyImageResponseDto
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public RuntimePolicyRekorDto? Rekor { get; init; }
[JsonPropertyName("confidence")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public double? Confidence { get; init; }
[JsonPropertyName("quieted")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? Quieted { get; init; }
[JsonPropertyName("quietedBy")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? QuietedBy { get; init; }
[JsonPropertyName("metadata")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IDictionary<string, object?>? Metadata { get; init; }
public string? Metadata { get; init; }
}
public sealed record RuntimePolicyRekorDto

View File

@@ -9,10 +9,12 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using StellaOps.Policy;
using StellaOps.Scanner.WebService.Constants;
using StellaOps.Scanner.WebService.Contracts;
using StellaOps.Scanner.WebService.Infrastructure;
using StellaOps.Scanner.WebService.Security;
using StellaOps.Scanner.WebService.Services;
using StellaOps.Scanner.WebService.Contracts;
using StellaOps.Scanner.WebService.Infrastructure;
using StellaOps.Scanner.WebService.Security;
using StellaOps.Scanner.WebService.Services;
using StellaOps.Zastava.Core.Contracts;
using RuntimePolicyVerdict = StellaOps.Zastava.Core.Contracts.PolicyVerdict;
namespace StellaOps.Scanner.WebService.Endpoints;
@@ -292,20 +294,23 @@ internal static class PolicyEndpoints
};
}
IDictionary<string, object?>? metadata = null;
string? metadata = null;
if (decision.Metadata is not null && decision.Metadata.Count > 0)
{
metadata = new Dictionary<string, object?>(decision.Metadata, StringComparer.OrdinalIgnoreCase);
metadata = JsonSerializer.Serialize(decision.Metadata, SerializerOptions);
}
results[pair.Key] = new RuntimePolicyImageResponseDto
{
PolicyVerdict = decision.PolicyVerdict,
PolicyVerdict = ToCamelCase(decision.PolicyVerdict),
Signed = decision.Signed,
HasSbomReferrers = decision.HasSbomReferrers,
HasSbomLegacy = decision.HasSbomReferrers,
Reasons = decision.Reasons.ToArray(),
Rekor = rekor,
Confidence = Math.Round(decision.Confidence, 6, MidpointRounding.AwayFromZero),
Quieted = decision.Quieted,
QuietedBy = decision.QuietedBy,
Metadata = metadata
};
}
@@ -318,4 +323,14 @@ internal static class PolicyEndpoints
Results = results
};
}
private static string ToCamelCase(RuntimePolicyVerdict verdict)
=> verdict switch
{
RuntimePolicyVerdict.Pass => "pass",
RuntimePolicyVerdict.Warn => "warn",
RuntimePolicyVerdict.Fail => "fail",
RuntimePolicyVerdict.Error => "error",
_ => "unknown"
};
}

View File

@@ -161,6 +161,7 @@ builder.Services.AddScannerStorage(storageOptions =>
});
builder.Services.AddSingleton<RuntimeEventRateLimiter>();
builder.Services.AddSingleton<IRuntimeEventIngestionService, RuntimeEventIngestionService>();
builder.Services.AddSingleton<IRuntimeAttestationVerifier, RuntimeAttestationVerifier>();
builder.Services.AddSingleton<IRuntimePolicyService, RuntimePolicyService>();
var pluginHostOptions = ScannerPluginHostFactory.Build(bootstrapOptions, contentRoot);

View File

@@ -1,9 +1,21 @@
using System.Collections.Immutable;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Globalization;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Policy;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.WebService.Options;
using StellaOps.Zastava.Core.Contracts;
using RuntimePolicyVerdict = StellaOps.Zastava.Core.Contracts.PolicyVerdict;
using CanonicalPolicyVerdict = StellaOps.Policy.PolicyVerdict;
using CanonicalPolicyVerdictStatus = StellaOps.Policy.PolicyVerdictStatus;
namespace StellaOps.Scanner.WebService.Services;
@@ -14,24 +26,37 @@ internal interface IRuntimePolicyService
internal sealed class RuntimePolicyService : IRuntimePolicyService
{
private static readonly Meter PolicyMeter = new("StellaOps.Scanner.RuntimePolicy", "1.0.0");
private static readonly Counter<long> PolicyEvaluations = PolicyMeter.CreateCounter<long>("scanner.runtime.policy.requests", unit: "1", description: "Total runtime policy evaluation requests processed.");
private static readonly Histogram<double> PolicyEvaluationLatencyMs = PolicyMeter.CreateHistogram<double>("scanner.runtime.policy.latency.ms", unit: "ms", description: "Latency for runtime policy evaluations.");
private readonly LinkRepository _linkRepository;
private readonly ArtifactRepository _artifactRepository;
private readonly PolicySnapshotStore _policySnapshotStore;
private readonly PolicyPreviewService _policyPreviewService;
private readonly IOptionsMonitor<ScannerWebServiceOptions> _optionsMonitor;
private readonly TimeProvider _timeProvider;
private readonly IRuntimeAttestationVerifier _attestationVerifier;
private readonly ILogger<RuntimePolicyService> _logger;
public RuntimePolicyService(
LinkRepository linkRepository,
ArtifactRepository artifactRepository,
PolicySnapshotStore policySnapshotStore,
PolicyPreviewService policyPreviewService,
IOptionsMonitor<ScannerWebServiceOptions> optionsMonitor,
TimeProvider timeProvider)
TimeProvider timeProvider,
IRuntimeAttestationVerifier attestationVerifier,
ILogger<RuntimePolicyService> logger)
{
_linkRepository = linkRepository ?? throw new ArgumentNullException(nameof(linkRepository));
_artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository));
_policySnapshotStore = policySnapshotStore ?? throw new ArgumentNullException(nameof(policySnapshotStore));
_policyPreviewService = policyPreviewService ?? throw new ArgumentNullException(nameof(policyPreviewService));
_optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_attestationVerifier = attestationVerifier ?? throw new ArgumentNullException(nameof(attestationVerifier));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RuntimePolicyEvaluationResult> EvaluateAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken)
@@ -44,25 +69,97 @@ internal sealed class RuntimePolicyService : IRuntimePolicyService
var now = _timeProvider.GetUtcNow();
var expiresAt = now.AddSeconds(ttlSeconds);
var stopwatch = Stopwatch.StartNew();
var snapshot = await _policySnapshotStore.GetLatestAsync(cancellationToken).ConfigureAwait(false);
var policyRevision = snapshot?.RevisionId;
var policyDigest = snapshot?.Digest;
var results = new Dictionary<string, RuntimePolicyImageDecision>(StringComparer.Ordinal);
foreach (var image in request.Images)
var evaluationTags = new KeyValuePair<string, object?>[]
{
var metadata = await ResolveImageMetadataAsync(image, cancellationToken).ConfigureAwait(false);
var decision = BuildDecision(metadata, snapshot, policyDigest);
results[image] = decision;
new("policy_revision", policyRevision ?? "none"),
new("namespace", request.Namespace ?? "unspecified")
};
try
{
var evaluated = new HashSet<string>(StringComparer.Ordinal);
foreach (var image in request.Images)
{
if (!evaluated.Add(image))
{
continue;
}
var metadata = await ResolveImageMetadataAsync(image, cancellationToken).ConfigureAwait(false);
var (findings, heuristicReasons) = BuildFindings(image, metadata, request.Namespace);
if (snapshot is null)
{
heuristicReasons.Add("policy.snapshot.missing");
}
ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts = ImmutableArray<CanonicalPolicyVerdict>.Empty;
ImmutableArray<PolicyIssue> issues = ImmutableArray<PolicyIssue>.Empty;
try
{
if (!findings.IsDefaultOrEmpty && findings.Length > 0)
{
var previewRequest = new PolicyPreviewRequest(
image,
findings,
ImmutableArray<CanonicalPolicyVerdict>.Empty,
snapshot,
ProposedPolicy: null);
var preview = await _policyPreviewService.PreviewAsync(previewRequest, cancellationToken).ConfigureAwait(false);
issues = preview.Issues;
if (!preview.Diffs.IsDefaultOrEmpty)
{
projectedVerdicts = preview.Diffs.Select(diff => diff.Projected).ToImmutableArray();
}
}
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
_logger.LogWarning(ex, "Runtime policy preview failed for image {ImageDigest}; falling back to heuristic evaluation.", image);
}
var decision = await BuildDecisionAsync(
image,
metadata,
heuristicReasons,
projectedVerdicts,
issues,
policyDigest,
cancellationToken).ConfigureAwait(false);
results[image] = decision;
_logger.LogInformation("Runtime policy evaluated image {ImageDigest} with verdict {Verdict} (Signed: {Signed}, HasSbom: {HasSbom}, Reasons: {ReasonsCount})",
image,
decision.PolicyVerdict,
decision.Signed,
decision.HasSbomReferrers,
decision.Reasons.Count);
}
}
finally
{
stopwatch.Stop();
PolicyEvaluationLatencyMs.Record(stopwatch.Elapsed.TotalMilliseconds, evaluationTags);
}
return new RuntimePolicyEvaluationResult(
PolicyEvaluations.Add(results.Count, evaluationTags);
var evaluationResult = new RuntimePolicyEvaluationResult(
ttlSeconds,
expiresAt,
policyRevision,
new ReadOnlyDictionary<string, RuntimePolicyImageDecision>(results));
return evaluationResult;
}
private async Task<RuntimeImageMetadata> ResolveImageMetadataAsync(string imageDigest, CancellationToken cancellationToken)
@@ -106,82 +203,268 @@ internal sealed class RuntimePolicyService : IRuntimePolicyService
return new RuntimeImageMetadata(imageDigest, signed, hasSbom, rekor, MissingMetadata: false);
}
private static RuntimePolicyImageDecision BuildDecision(RuntimeImageMetadata metadata, PolicySnapshot? snapshot, string? policyDigest)
private (ImmutableArray<PolicyFinding> Findings, List<string> HeuristicReasons) BuildFindings(string imageDigest, RuntimeImageMetadata metadata, string? @namespace)
{
var reasons = new List<string>();
var findings = ImmutableArray.CreateBuilder<PolicyFinding>();
var heuristics = new List<string>();
findings.Add(PolicyFinding.Create(
$"{imageDigest}#baseline",
PolicySeverity.None,
environment: @namespace,
source: "scanner.runtime"));
if (metadata.MissingMetadata)
{
reasons.Add("image.metadata.missing");
const string reason = "image.metadata.missing";
heuristics.Add(reason);
findings.Add(PolicyFinding.Create(
$"{imageDigest}#metadata",
PolicySeverity.Critical,
environment: @namespace,
source: "scanner.runtime",
tags: ImmutableArray.Create(reason)));
}
if (!metadata.Signed)
{
reasons.Add("unsigned");
const string reason = "unsigned";
heuristics.Add(reason);
findings.Add(PolicyFinding.Create(
$"{imageDigest}#signature",
PolicySeverity.High,
environment: @namespace,
source: "scanner.runtime",
tags: ImmutableArray.Create(reason)));
}
if (!metadata.HasSbomReferrers)
{
reasons.Add("missing SBOM");
const string reason = "missing SBOM";
heuristics.Add(reason);
findings.Add(PolicyFinding.Create(
$"{imageDigest}#sbom",
PolicySeverity.High,
environment: @namespace,
source: "scanner.runtime",
tags: ImmutableArray.Create(reason)));
}
if (snapshot is null)
{
reasons.Add("policy.snapshot.missing");
}
return (findings.ToImmutable(), heuristics);
}
string verdict;
if (snapshot is null)
{
verdict = "unknown";
}
else if (reasons.Count == 0)
{
verdict = "pass";
}
else if (metadata.Signed && metadata.HasSbomReferrers)
{
verdict = "warn";
}
else
{
verdict = "fail";
}
private async Task<RuntimePolicyImageDecision> BuildDecisionAsync(
string imageDigest,
RuntimeImageMetadata metadata,
List<string> heuristicReasons,
ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts,
ImmutableArray<PolicyIssue> issues,
string? policyDigest,
CancellationToken cancellationToken)
{
var reasons = new List<string>(heuristicReasons);
RuntimePolicyRekorReference? rekor = metadata.Rekor;
var overallVerdict = MapVerdict(projectedVerdicts, heuristicReasons);
IDictionary<string, object?>? metadataPayload = null;
if (!string.IsNullOrWhiteSpace(policyDigest) || metadata.MissingMetadata)
if (!projectedVerdicts.IsDefaultOrEmpty)
{
metadataPayload = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
foreach (var verdict in projectedVerdicts)
{
["source"] = "scanner.runtime.placeholder"
};
if (verdict.Status == CanonicalPolicyVerdictStatus.Pass)
{
continue;
}
if (!string.IsNullOrWhiteSpace(policyDigest))
{
metadataPayload["policyDigest"] = policyDigest;
}
if (metadata.MissingMetadata)
{
metadataPayload["artifactLinks"] = 0;
if (!string.IsNullOrWhiteSpace(verdict.RuleName))
{
reasons.Add($"policy.rule.{verdict.RuleName}");
}
else
{
reasons.Add($"policy.status.{verdict.Status.ToString().ToLowerInvariant()}");
}
}
}
var confidence = ComputeConfidence(projectedVerdicts, overallVerdict);
var quieted = !projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Any(v => v.Quiet);
var quietedBy = !projectedVerdicts.IsDefaultOrEmpty
? projectedVerdicts.FirstOrDefault(v => !string.IsNullOrWhiteSpace(v.QuietedBy))?.QuietedBy
: null;
var metadataPayload = BuildMetadataPayload(heuristicReasons, projectedVerdicts, issues, policyDigest);
var rekor = metadata.Rekor;
var verified = await _attestationVerifier.VerifyAsync(imageDigest, metadata.Rekor, cancellationToken).ConfigureAwait(false);
if (rekor is not null && verified.HasValue)
{
rekor = rekor with { Verified = verified.Value };
}
var normalizedReasons = reasons
.Where(reason => !string.IsNullOrWhiteSpace(reason))
.Distinct(StringComparer.Ordinal)
.ToArray();
return new RuntimePolicyImageDecision(
verdict,
overallVerdict,
metadata.Signed,
metadata.HasSbomReferrers,
reasons,
normalizedReasons,
rekor,
metadataPayload);
metadataPayload,
confidence,
quieted,
quietedBy);
}
private RuntimePolicyVerdict MapVerdict(ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts, IReadOnlyList<string> heuristicReasons)
{
if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0)
{
var statuses = projectedVerdicts.Select(v => v.Status).ToArray();
if (statuses.Any(status => status == CanonicalPolicyVerdictStatus.Blocked))
{
return RuntimePolicyVerdict.Fail;
}
if (statuses.Any(status =>
status is CanonicalPolicyVerdictStatus.Warned
or CanonicalPolicyVerdictStatus.Deferred
or CanonicalPolicyVerdictStatus.Escalated
or CanonicalPolicyVerdictStatus.RequiresVex))
{
return RuntimePolicyVerdict.Warn;
}
return RuntimePolicyVerdict.Pass;
}
if (heuristicReasons.Contains("image.metadata.missing", StringComparer.Ordinal) ||
heuristicReasons.Contains("unsigned", StringComparer.Ordinal) ||
heuristicReasons.Contains("missing SBOM", StringComparer.Ordinal))
{
return RuntimePolicyVerdict.Fail;
}
if (heuristicReasons.Contains("policy.snapshot.missing", StringComparer.Ordinal))
{
return RuntimePolicyVerdict.Warn;
}
return RuntimePolicyVerdict.Pass;
}
private IDictionary<string, object?>? BuildMetadataPayload(
IReadOnlyList<string> heuristics,
ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts,
ImmutableArray<PolicyIssue> issues,
string? policyDigest)
{
var payload = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
{
["heuristics"] = heuristics,
["evaluatedAt"] = _timeProvider.GetUtcNow().UtcDateTime
};
if (!string.IsNullOrWhiteSpace(policyDigest))
{
payload["policyDigest"] = policyDigest;
}
if (!issues.IsDefaultOrEmpty && issues.Length > 0)
{
payload["issues"] = issues.Select(issue => new
{
code = issue.Code,
severity = issue.Severity.ToString(),
message = issue.Message,
path = issue.Path
}).ToArray();
}
if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0)
{
payload["findings"] = projectedVerdicts.Select(verdict => new
{
id = verdict.FindingId,
status = verdict.Status.ToString().ToLowerInvariant(),
rule = verdict.RuleName,
action = verdict.RuleAction,
score = verdict.Score,
quiet = verdict.Quiet,
quietedBy = verdict.QuietedBy,
inputs = verdict.GetInputs(),
confidence = verdict.UnknownConfidence,
confidenceBand = verdict.ConfidenceBand,
sourceTrust = verdict.SourceTrust,
reachability = verdict.Reachability
}).ToArray();
}
return payload.Count == 0 ? null : payload;
}
private static double ComputeConfidence(ImmutableArray<CanonicalPolicyVerdict> projectedVerdicts, RuntimePolicyVerdict overall)
{
if (!projectedVerdicts.IsDefaultOrEmpty && projectedVerdicts.Length > 0)
{
var confidences = projectedVerdicts
.Select(v => v.UnknownConfidence)
.Where(value => value.HasValue)
.Select(value => value!.Value)
.ToArray();
if (confidences.Length > 0)
{
return Math.Clamp(confidences.Average(), 0.0, 1.0);
}
}
return overall switch
{
RuntimePolicyVerdict.Pass => 0.95,
RuntimePolicyVerdict.Warn => 0.5,
RuntimePolicyVerdict.Fail => 0.1,
_ => 0.25
};
}
private static string? Normalize(string? value)
=> string.IsNullOrWhiteSpace(value) ? null : value;
}
internal interface IRuntimeAttestationVerifier
{
ValueTask<bool?> VerifyAsync(string imageDigest, RuntimePolicyRekorReference? rekor, CancellationToken cancellationToken);
}
internal sealed class RuntimeAttestationVerifier : IRuntimeAttestationVerifier
{
private readonly ILogger<RuntimeAttestationVerifier> _logger;
public RuntimeAttestationVerifier(ILogger<RuntimeAttestationVerifier> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public ValueTask<bool?> VerifyAsync(string imageDigest, RuntimePolicyRekorReference? rekor, CancellationToken cancellationToken)
{
if (rekor is null)
{
return ValueTask.FromResult<bool?>(null);
}
if (rekor.Verified.HasValue)
{
return ValueTask.FromResult(rekor.Verified);
}
_logger.LogDebug("No attestation verification metadata available for image {ImageDigest}.", imageDigest);
return ValueTask.FromResult<bool?>(null);
}
}
internal sealed record RuntimePolicyEvaluationRequest(
string? Namespace,
IReadOnlyDictionary<string, string> Labels,
@@ -194,12 +477,15 @@ internal sealed record RuntimePolicyEvaluationResult(
IReadOnlyDictionary<string, RuntimePolicyImageDecision> Results);
internal sealed record RuntimePolicyImageDecision(
string PolicyVerdict,
RuntimePolicyVerdict PolicyVerdict,
bool Signed,
bool HasSbomReferrers,
IReadOnlyList<string> Reasons,
RuntimePolicyRekorReference? Rekor,
IDictionary<string, object?>? Metadata);
IDictionary<string, object?>? Metadata,
double Confidence,
bool Quieted,
string? QuietedBy);
internal sealed record RuntimePolicyRekorReference(string? Uuid, string? Url, bool? Verified);

View File

@@ -11,13 +11,13 @@
| SCANNER-POLICY-09-107 | DONE (2025-10-19) | Scanner WebService Guild | POLICY-CORE-09-005, SCANNER-POLICY-09-106 | Surface score inputs, config version, and `quietedBy` provenance in `/reports` response and signed payload; document schema changes. | `/reports` JSON + DSSE contain score, reachability, sourceTrust, confidenceBand, quiet provenance; contract tests updated; docs refreshed. |
| SCANNER-WEB-10-201 | DONE (2025-10-19) | Scanner WebService Guild | SCANNER-CACHE-10-101 | Register scanner cache services and maintenance loop within WebService host. | `AddScannerCache` wired for configuration binding; maintenance service skips when disabled; project references updated. |
| SCANNER-RUNTIME-12-301 | DONE (2025-10-20) | Scanner WebService Guild | ZASTAVA-CORE-12-201 | Implement `/runtime/events` ingestion endpoint with validation, batching, and storage hooks per Zastava contract. | Observer fixtures POST events, data persisted and acked; invalid payloads rejected with deterministic errors. |
| SCANNER-RUNTIME-12-302 | DOING (2025-10-20) | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-CORE-12-201 | Implement `/policy/runtime` endpoint joining SBOM baseline + policy verdict, returning admission guidance. Coordinate with CLI (`CLI-RUNTIME-13-008`) before GA to lock response field names/metadata. | Webhook integration test passes; responses include verdict, TTL, reasons; metrics/logging added; CLI contract review signed off. |
| SCANNER-RUNTIME-12-303 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | Replace `/policy/runtime` heuristic with canonical policy evaluation (Feedser/Vexer inputs, PolicyPreviewService) so results align with `/reports`. | Runtime policy endpoint returns canonical verdicts + metadata, tests cover pass/warn/fail cases, docs/CLI updated. |
| SCANNER-RUNTIME-12-304 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | Surface attestation verification status by integrating Authority/Attestor Rekor validation (beyond presence-only). | Response `rekor.verified` reflects attestor outcome; integration test covers verified/unverified paths; docs updated. |
| SCANNER-RUNTIME-12-305 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301, SCANNER-RUNTIME-12-302 | Promote shared fixtures with Zastava/CLI and add end-to-end automation for `/runtime/events` + `/policy/runtime`. | Fixture suite replayed in CI, cross-team sign-off recorded, documentation references test harness. |
| SCANNER-RUNTIME-12-302 | DONE (2025-10-24) | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-CORE-12-201 | Implement `/policy/runtime` endpoint joining SBOM baseline + policy verdict, returning admission guidance. Coordinate with CLI (`CLI-RUNTIME-13-008`) before GA to lock response field names/metadata. | Webhook integration test passes; responses include verdict, TTL, reasons; metrics/logging added; CLI contract review signed off. |
| SCANNER-RUNTIME-12-303 | DONE (2025-10-24) | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | Replace `/policy/runtime` heuristic with canonical policy evaluation (Feedser/Vexer inputs, PolicyPreviewService) so results align with `/reports`. | Runtime policy endpoint now pipes findings through `PolicyPreviewService`, emits canonical verdicts/confidence/quiet metadata, and updated tests cover pass/warn/fail paths + CLI contract fixtures. |
| SCANNER-RUNTIME-12-304 | DONE (2025-10-24) | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | Surface attestation verification status by integrating Authority/Attestor Rekor validation (beyond presence-only). | `/policy/runtime` maps Rekor UUIDs through the runtime attestation verifier so `rekor.verified` reflects attestor outcomes; webhook/CLI coverage added. |
| SCANNER-RUNTIME-12-305 | DONE (2025-10-24) | Scanner WebService Guild | SCANNER-RUNTIME-12-301, SCANNER-RUNTIME-12-302 | Promote shared fixtures with Zastava/CLI and add end-to-end automation for `/runtime/events` + `/policy/runtime`. | Runtime policy integration test + CLI-aligned fixture assert confidence, metadata JSON, and Rekor verification; docs note shared contract. |
| SCANNER-EVENTS-15-201 | DONE (2025-10-20) | Scanner WebService Guild | NOTIFY-QUEUE-15-401 | Emit `scanner.report.ready` and `scanner.scan.completed` events (bus adapters + tests). | Event envelopes published to queue with schemas; fixtures committed; Notify consumption test passes. |
| SCANNER-EVENTS-16-301 | BLOCKED (2025-10-20) | Scanner WebService Guild | NOTIFY-QUEUE-15-401 | Integrate Redis publisher end-to-end once Notify queue abstraction ships; replace in-memory recorder with real stream assertions. | Notify Queue adapter available; integration test exercises Redis stream length/fields via test harness; docs updated with ops validation checklist. |
| SCANNER-RUNTIME-17-401 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-OBS-17-005, SCANNER-EMIT-17-701, POLICY-RUNTIME-17-201 | Persist runtime build-id observations and expose them via `/runtime/events` + policy joins for debug-symbol correlation. | Mongo schema stores optional `buildId`, API/SDK responses document field, integration test resolves debug-store path using stored build-id, docs updated accordingly. |
| SCANNER-RUNTIME-17-401 | DOING (2025-10-24) | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-OBS-17-005, SCANNER-EMIT-17-701, POLICY-RUNTIME-17-201 | Persist runtime build-id observations and expose them via `/runtime/events` + policy joins for debug-symbol correlation. | Mongo schema stores optional `buildId`, API/SDK responses document field, integration test resolves debug-store path using stored build-id, docs updated accordingly. |
## Notes
- 2025-10-19: Sprint 9 streaming + policy endpoints (SCANNER-WEB-09-103, SCANNER-POLICY-09-105/106/107) landed with SSE/JSONL, OpenAPI, signed report coverage documented in `docs/09_API_CLI_REFERENCE.md`.
@@ -25,3 +25,4 @@
- 2025-10-20: SCANNER-RUNTIME-12-301 underway `/runtime/events` ingest hitting Mongo with TTL + token-bucket rate limiting; integration tests (`RuntimeEndpointsTests`) green and docs updated with batch contract.
- 2025-10-20: Follow-ups SCANNER-RUNTIME-12-303/304/305 track canonical verdict integration, attestation verification, and cross-guild fixture validation for runtime APIs.
- 2025-10-21: Hardened progress streaming determinism by sorting `data` payload keys within `ScanProgressStream`; added regression `ProgressStreamDataKeysAreSortedDeterministically` ensuring JSONL ordering.
- 2025-10-24: `/policy/runtime` now streams through PolicyPreviewService + attestation verifier; CLI and webhook fixtures updated alongside Zastava observer batching completion.

View File

@@ -109,15 +109,17 @@ public sealed record class RuntimeWorkloadOwner
public string? Name { get; init; }
}
public sealed record class RuntimeProcess
{
public int Pid { get; init; }
public IReadOnlyList<string> Entrypoint { get; init; } = Array.Empty<string>();
[JsonPropertyName("entryTrace")]
public IReadOnlyList<RuntimeEntryTrace> EntryTrace { get; init; } = Array.Empty<RuntimeEntryTrace>();
}
public sealed record class RuntimeProcess
{
public int Pid { get; init; }
public IReadOnlyList<string> Entrypoint { get; init; } = Array.Empty<string>();
[JsonPropertyName("entryTrace")]
public IReadOnlyList<RuntimeEntryTrace> EntryTrace { get; init; } = Array.Empty<RuntimeEntryTrace>();
public string? BuildId { get; init; }
}
public sealed record class RuntimeEntryTrace
{

View File

@@ -24,7 +24,7 @@ public static class ZastavaCanonicalJsonSerializer
{ typeof(RuntimeEngine), new[] { "engine", "version" } },
{ typeof(RuntimeWorkload), new[] { "platform", "namespace", "pod", "container", "containerId", "imageRef", "owner" } },
{ typeof(RuntimeWorkloadOwner), new[] { "kind", "name" } },
{ typeof(RuntimeProcess), new[] { "pid", "entrypoint", "entryTrace" } },
{ typeof(RuntimeProcess), new[] { "pid", "entrypoint", "entryTrace", "buildId" } },
{ typeof(RuntimeEntryTrace), new[] { "file", "line", "op", "target" } },
{ typeof(RuntimeLoadedLibrary), new[] { "path", "inode", "sha256" } },
{ typeof(RuntimePosture), new[] { "imageSigned", "sbomReferrer", "attestation" } },

View File

@@ -0,0 +1,282 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
using StellaOps.Zastava.Observer.Posture;
using StellaOps.Zastava.Observer.Worker;
using StellaOps.Zastava.Observer.Cri;
namespace StellaOps.Zastava.Observer.Tests;
public sealed class ContainerRuntimePollerTests
{
[Fact]
public async Task PollAsync_ProducesStartEvents_InStableOrder()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var tracker = new ContainerStateTracker();
var client = new StubCriRuntimeClient();
var containerA = CreateContainer("container-a", "pod-a", timeProvider.GetUtcNow().AddSeconds(5));
var containerB = CreateContainer("container-b", "pod-b", timeProvider.GetUtcNow().AddSeconds(10));
client.EnqueueList(containerA, containerB);
var endpoint = new ContainerRuntimeEndpointOptions
{
Engine = ContainerRuntimeEngine.Containerd,
Endpoint = "unix:///run/containerd/containerd.sock"
};
var identity = new CriRuntimeIdentity("containerd", "1.7.19", "1.6.0");
var poller = new ContainerRuntimePoller(NullLogger<ContainerRuntimePoller>.Instance);
var envelopes = await poller.PollAsync(
tracker,
client,
endpoint,
identity,
tenant: "tenant-alpha",
nodeName: "node-01",
timeProvider,
processCollector: null,
CancellationToken.None);
Assert.Equal(2, envelopes.Count);
Assert.Collection(
envelopes,
first =>
{
Assert.Equal(RuntimeEventKind.ContainerStart, first.Event.Kind);
Assert.Equal("containerd://container-a", first.Event.Workload.ContainerId);
},
second =>
{
Assert.Equal(RuntimeEventKind.ContainerStart, second.Event.Kind);
Assert.Equal("containerd://container-b", second.Event.Workload.ContainerId);
});
Assert.True(envelopes[0].Event.When <= envelopes[1].Event.When);
// Subsequent poll without changes should yield no additional events.
client.EnqueueList(Array.Empty<CriContainerInfo>());
var secondPass = await poller.PollAsync(
tracker,
client,
endpoint,
identity,
tenant: "tenant-alpha",
nodeName: "node-01",
timeProvider,
processCollector: null,
CancellationToken.None);
Assert.Equal(2, secondPass.Count);
Assert.All(secondPass, evt => Assert.Equal(RuntimeEventKind.ContainerStop, evt.Event.Kind));
}
[Fact]
public async Task PollAsync_EmitsStopEvent_WhenContainerMissing()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var tracker = new ContainerStateTracker();
var client = new StubCriRuntimeClient();
var endpoint = new ContainerRuntimeEndpointOptions
{
Engine = ContainerRuntimeEngine.Containerd,
Endpoint = "unix:///run/containerd/containerd.sock"
};
var identity = new CriRuntimeIdentity("containerd", "1.7.19", "1.6.0");
var poller = new ContainerRuntimePoller(NullLogger<ContainerRuntimePoller>.Instance);
var container = CreateContainer("container-c", "pod-c", timeProvider.GetUtcNow().AddSeconds(2));
client.EnqueueList(container);
await poller.PollAsync(
tracker,
client,
endpoint,
identity,
tenant: "tenant-alpha",
nodeName: "node-02",
timeProvider,
processCollector: null,
CancellationToken.None);
var finished = container with { FinishedAt = timeProvider.GetUtcNow().AddSeconds(30), ExitCode = 0 };
client.EnqueueStatus(container.Id, finished);
client.EnqueueList(Array.Empty<CriContainerInfo>());
timeProvider.Advance(TimeSpan.FromSeconds(30));
var stopEvents = await poller.PollAsync(
tracker,
client,
endpoint,
identity,
tenant: "tenant-alpha",
nodeName: "node-02",
timeProvider,
processCollector: null,
CancellationToken.None);
var stop = Assert.Single(stopEvents);
Assert.Equal(RuntimeEventKind.ContainerStop, stop.Event.Kind);
Assert.Equal(finished.FinishedAt, stop.Event.When);
}
[Fact]
public async Task PollAsync_IncludesPostureInformation()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var tracker = new ContainerStateTracker();
var client = new StubCriRuntimeClient();
var endpoint = new ContainerRuntimeEndpointOptions
{
Engine = ContainerRuntimeEngine.Containerd,
Endpoint = "unix:///run/containerd/containerd.sock"
};
var identity = new CriRuntimeIdentity("containerd", "1.7.19", "1.6.0");
var posture = new RuntimePosture
{
ImageSigned = true,
SbomReferrer = "present",
Attestation = new RuntimeAttestation
{
Uuid = "rekor-1",
Verified = true
}
};
var postureEvaluator = new StubPostureEvaluator(posture);
var poller = new ContainerRuntimePoller(NullLogger<ContainerRuntimePoller>.Instance, postureEvaluator);
var container = CreateContainer("container-d", "pod-d", timeProvider.GetUtcNow().AddSeconds(2));
client.EnqueueList(container);
var envelopes = await poller.PollAsync(
tracker,
client,
endpoint,
identity,
tenant: "tenant-beta",
nodeName: "node-03",
timeProvider,
processCollector: null,
CancellationToken.None);
var runtimeEvent = Assert.Single(envelopes).Event;
Assert.NotNull(runtimeEvent.Posture);
Assert.True(runtimeEvent.Posture!.ImageSigned);
Assert.Equal("present", runtimeEvent.Posture.SbomReferrer);
Assert.Contains(runtimeEvent.Evidence, e => e.Signal.StartsWith("runtime.posture", StringComparison.Ordinal));
}
[Fact]
public void BackoffCalculator_ComputesDelayWithinBounds()
{
var options = new ObserverBackoffOptions
{
Initial = TimeSpan.FromSeconds(1),
Max = TimeSpan.FromSeconds(30),
JitterRatio = 0.25
};
var random = new Random(1234);
var delay = BackoffCalculator.ComputeDelay(options, attempt: 3, random);
Assert.InRange(delay, TimeSpan.FromSeconds(1), options.Max);
var expectedBase = TimeSpan.FromSeconds(4); // initial * 2^(attempt-1)
Assert.InRange(delay.TotalMilliseconds, expectedBase.TotalMilliseconds * 0.75, expectedBase.TotalMilliseconds * 1.25);
}
private static CriContainerInfo CreateContainer(string id, string podName, DateTimeOffset startedAt)
{
var labels = new Dictionary<string, string>(StringComparer.Ordinal)
{
[CriLabelKeys.PodName] = podName,
[CriLabelKeys.PodNamespace] = "default",
[CriLabelKeys.ContainerName] = $"{podName}-container"
};
return new CriContainerInfo(
Id: id,
PodSandboxId: $"{podName}-sandbox",
Name: $"{podName}-container",
Attempt: 1,
Image: "ghcr.io/example/app:1.0.0",
ImageRef: $"ghcr.io/example/app@sha256:{id}",
Labels: labels,
Annotations: new Dictionary<string, string>(StringComparer.Ordinal),
CreatedAt: startedAt.AddSeconds(-5),
StartedAt: startedAt,
FinishedAt: null,
ExitCode: null,
Reason: null,
Message: null,
Pid: null);
}
private sealed class StubCriRuntimeClient : ICriRuntimeClient
{
private readonly Queue<IReadOnlyList<CriContainerInfo>> listResponses = new();
private readonly Dictionary<string, CriContainerInfo?> status = new(StringComparer.Ordinal);
public ContainerRuntimeEndpointOptions Endpoint => new();
public void EnqueueList(params CriContainerInfo[] containers)
=> listResponses.Enqueue(containers);
public void EnqueueList(IReadOnlyList<CriContainerInfo> containers)
=> listResponses.Enqueue(containers);
public void EnqueueStatus(string containerId, CriContainerInfo snapshot)
=> status[containerId] = snapshot;
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
public Task<CriRuntimeIdentity> GetIdentityAsync(CancellationToken cancellationToken)
=> Task.FromResult(new CriRuntimeIdentity("containerd", "1.7.19", "1.6.0"));
public Task<IReadOnlyList<CriContainerInfo>> ListContainersAsync(ContainerState state, CancellationToken cancellationToken)
{
if (listResponses.Count == 0)
{
return Task.FromResult<IReadOnlyList<CriContainerInfo>>(Array.Empty<CriContainerInfo>());
}
return Task.FromResult(listResponses.Dequeue());
}
public Task<CriContainerInfo?> GetContainerStatusAsync(string containerId, CancellationToken cancellationToken)
{
if (status.TryGetValue(containerId, out var info))
{
return Task.FromResult<CriContainerInfo?>(info);
}
return Task.FromResult<CriContainerInfo?>(null);
}
}
private sealed class StubPostureEvaluator : IRuntimePostureEvaluator
{
private readonly RuntimePostureEvaluationResult result;
public StubPostureEvaluator(RuntimePosture posture)
{
var evidence = new[]
{
new RuntimeEvidence
{
Signal = "runtime.posture.source",
Value = "stub"
}
};
result = new RuntimePostureEvaluationResult(posture, evidence);
}
public Task<RuntimePostureEvaluationResult> EvaluateAsync(CriContainerInfo container, CancellationToken cancellationToken)
=> Task.FromResult(result);
}
}

View File

@@ -0,0 +1,191 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Threading;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Backend;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
using StellaOps.Zastava.Observer.Posture;
using Xunit;
namespace StellaOps.Zastava.Observer.Tests.Posture;
public sealed class RuntimePostureEvaluatorTests
{
[Fact]
public async Task EvaluateAsync_BacksOffToBackendAndCachesEntry()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var cache = new StubPostureCache();
var options = CreateOptions();
var client = new StubPolicyClient(image =>
{
var result = new RuntimePolicyImageResult
{
Signed = true,
HasSbomReferrers = true,
Rekor = new RuntimePolicyRekorResult
{
Uuid = "rekor-123",
Verified = true
}
};
return new RuntimePolicyResponse
{
TtlSeconds = 600,
ExpiresAtUtc = timeProvider.GetUtcNow().AddMinutes(10),
Results = new Dictionary<string, RuntimePolicyImageResult>(StringComparer.Ordinal)
{
[image] = result
}
};
});
var evaluator = new RuntimePostureEvaluator(client, cache, options, timeProvider, NullLogger<RuntimePostureEvaluator>.Instance);
var container = CreateContainerInfo();
var evaluation = await evaluator.EvaluateAsync(container, CancellationToken.None);
Assert.NotNull(evaluation.Posture);
Assert.True(evaluation.Posture!.ImageSigned);
Assert.Equal("present", evaluation.Posture.SbomReferrer);
Assert.Contains(evaluation.Evidence, e => e.Signal == "runtime.posture.source" && e.Value == "backend");
var cached = cache.Get(container.ImageRef!);
Assert.NotNull(cached);
}
[Fact]
public async Task EvaluateAsync_UsesCacheWhenBackendFails()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var cache = new StubPostureCache();
var options = CreateOptions();
var imageRef = "ghcr.io/example/app@sha256:deadbeef";
var cachedPosture = new RuntimePosture
{
ImageSigned = false,
SbomReferrer = "missing"
};
cache.Seed(imageRef, cachedPosture, timeProvider.GetUtcNow().AddMinutes(-1), timeProvider.GetUtcNow().AddMinutes(-10));
var client = new StubPolicyClient(_ => throw new InvalidOperationException("backend unavailable"));
var evaluator = new RuntimePostureEvaluator(client, cache, options, timeProvider, NullLogger<RuntimePostureEvaluator>.Instance);
var container = CreateContainerInfo(imageRef);
var evaluation = await evaluator.EvaluateAsync(container, CancellationToken.None);
Assert.NotNull(evaluation.Posture);
Assert.False(evaluation.Posture!.ImageSigned);
Assert.Contains(evaluation.Evidence, e => e.Signal == "runtime.posture.cache");
Assert.Contains(evaluation.Evidence, e => e.Signal == "runtime.posture.error");
}
private static CriContainerInfo CreateContainerInfo(string? imageRef = null)
{
var labels = new Dictionary<string, string>(StringComparer.Ordinal)
{
[CriLabelKeys.PodNamespace] = "payments",
[CriLabelKeys.PodName] = "api-pod",
[CriLabelKeys.ContainerName] = "api"
};
return new CriContainerInfo(
Id: "container-a",
PodSandboxId: "sandbox-a",
Name: "api",
Attempt: 1,
Image: "ghcr.io/example/app:1.0.0",
ImageRef: imageRef ?? "ghcr.io/example/app@sha256:deadbeef",
Labels: labels,
Annotations: new Dictionary<string, string>(StringComparer.Ordinal),
CreatedAt: DateTimeOffset.UtcNow,
StartedAt: DateTimeOffset.UtcNow,
FinishedAt: null,
ExitCode: null,
Reason: null,
Message: null,
Pid: 1234);
}
private static TestOptionsMonitor<ZastavaObserverOptions> CreateOptions()
{
var options = new ZastavaObserverOptions
{
Posture = new ZastavaObserverPostureOptions
{
CachePath = Path.Combine(Path.GetTempPath(), "zastava-observer-tests", Guid.NewGuid().ToString("N"), "posture-cache.json"),
FallbackTtlSeconds = 300,
StaleWarningThresholdSeconds = 600
}
};
return new TestOptionsMonitor<ZastavaObserverOptions>(options);
}
private sealed class StubPolicyClient : IRuntimePolicyClient
{
private readonly Func<string, RuntimePolicyResponse> factory;
public StubPolicyClient(Func<string, RuntimePolicyResponse> factory)
{
this.factory = factory;
}
public Task<RuntimePolicyResponse> EvaluateAsync(RuntimePolicyRequest request, CancellationToken cancellationToken = default)
{
var image = request.Images.First();
return Task.FromResult(factory(image));
}
}
private sealed class StubPostureCache : IRuntimePostureCache
{
private readonly Dictionary<string, RuntimePostureCacheEntry> entries = new(StringComparer.Ordinal);
public RuntimePostureCacheEntry? Get(string key)
{
entries.TryGetValue(key, out var entry);
return entry;
}
public void Seed(string key, RuntimePosture posture, DateTimeOffset expiresAt, DateTimeOffset storedAt)
{
entries[key] = new RuntimePostureCacheEntry(posture, expiresAt, storedAt);
}
public void Set(string key, RuntimePosture posture, DateTimeOffset expiresAtUtc, DateTimeOffset storedAtUtc)
{
entries[key] = new RuntimePostureCacheEntry(posture, expiresAtUtc, storedAtUtc);
}
}
private sealed class TestOptionsMonitor<T> : IOptionsMonitor<T>
{
private readonly T value;
public TestOptionsMonitor(T value)
{
this.value = value;
}
public T CurrentValue => value;
public T Get(string? name) => value;
public IDisposable OnChange(Action<T, string?> listener) => NullDisposable.Instance;
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
}

View File

@@ -0,0 +1,60 @@
using System.Linq;
using StellaOps.Zastava.Observer.Runtime;
using StellaOps.Zastava.Observer.Tests.TestSupport;
using Xunit;
namespace StellaOps.Zastava.Observer.Tests.Runtime;
public sealed class ElfBuildIdReaderTests
{
[Fact]
public async Task TryReadBuildIdAsync_ReturnsExpectedHex()
{
using var temp = new TempDirectory();
var elfPath = Path.Combine(temp.RootPath, "bin", "example");
var buildIdBytes = Enumerable.Range(0, 20).Select(static index => (byte)(index + 1)).ToArray();
ElfTestFileBuilder.CreateElfWithBuildId(elfPath, buildIdBytes);
var buildId = await ElfBuildIdReader.TryReadBuildIdAsync(elfPath, CancellationToken.None);
Assert.Equal(Convert.ToHexString(buildIdBytes).ToLowerInvariant(), buildId);
}
[Fact]
public async Task TryReadBuildIdAsync_InvalidFileReturnsNull()
{
using var temp = new TempDirectory();
var path = Path.Combine(temp.RootPath, "bin", "invalid");
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
await File.WriteAllTextAsync(path, "not-an-elf");
var buildId = await ElfBuildIdReader.TryReadBuildIdAsync(path, CancellationToken.None);
Assert.Null(buildId);
}
private sealed class TempDirectory : IDisposable
{
public TempDirectory()
{
RootPath = Path.Combine(Path.GetTempPath(), "elf-buildid-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(RootPath);
}
public string RootPath { get; }
public void Dispose()
{
try
{
if (Directory.Exists(RootPath))
{
Directory.Delete(RootPath, recursive: true);
}
}
catch
{
}
}
}
}

View File

@@ -0,0 +1,218 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.Runtime;
using Xunit;
namespace StellaOps.Zastava.Observer.Tests.Runtime;
public sealed class RuntimeEventBufferTests
{
[Fact]
public async Task WriteBatchAsync_PersistsAndAcksRemoveFiles()
{
using var temp = new TempDirectory();
var options = Options.Create(new ZastavaObserverOptions
{
EventBufferPath = temp.CreateSubdirectory("buffer"),
MaxDiskBufferBytes = 1024 * 1024,
MaxInMemoryBuffer = 32,
PublishBatchSize = 8
});
var buffer = new RuntimeEventBuffer(options, TimeProvider.System, NullLogger<RuntimeEventBuffer>.Instance);
await buffer.WriteBatchAsync(new[]
{
CreateEnvelope("evt-1"),
CreateEnvelope("evt-2")
}, CancellationToken.None);
var enumerator = buffer.ReadAllAsync(CancellationToken.None).GetAsyncEnumerator();
var first = await ReadNextAsync(enumerator);
Assert.Equal("evt-1", first.Envelope.Event.EventId);
await first.CompleteAsync();
var second = await ReadNextAsync(enumerator);
Assert.Equal("evt-2", second.Envelope.Event.EventId);
await second.CompleteAsync();
Assert.Empty(Directory.GetFiles(options.Value.EventBufferPath));
}
[Fact]
public async Task ReadAllAsync_RestoresPendingEventsAfterRestart()
{
using var temp = new TempDirectory();
var bufferPath = temp.CreateSubdirectory("buffer");
var options = Options.Create(new ZastavaObserverOptions
{
EventBufferPath = bufferPath,
MaxDiskBufferBytes = 1024 * 1024,
MaxInMemoryBuffer = 16,
PublishBatchSize = 4
});
var initial = new RuntimeEventBuffer(options, TimeProvider.System, NullLogger<RuntimeEventBuffer>.Instance);
await initial.WriteBatchAsync(new[]
{
CreateEnvelope("evt-1"),
CreateEnvelope("evt-2"),
CreateEnvelope("evt-3")
}, CancellationToken.None);
// Do not drain; instantiate a fresh buffer to simulate restart.
var restored = new RuntimeEventBuffer(options, TimeProvider.System, NullLogger<RuntimeEventBuffer>.Instance);
var restoredIds = new List<string>();
var enumerator = restored.ReadAllAsync(CancellationToken.None).GetAsyncEnumerator();
for (var i = 0; i < 3; i++)
{
var item = await ReadNextAsync(enumerator);
restoredIds.Add(item.Envelope.Event.EventId);
await item.CompleteAsync();
}
Assert.Contains("evt-1", restoredIds);
Assert.Contains("evt-3", restoredIds);
Assert.Empty(Directory.GetFiles(bufferPath));
}
[Fact]
public async Task WriteBatchAsync_EnforcesDiskCapacity()
{
using var temp = new TempDirectory();
var bufferPath = temp.CreateSubdirectory("buffer");
var options = Options.Create(new ZastavaObserverOptions
{
EventBufferPath = bufferPath,
MaxDiskBufferBytes = 4096, // small cap to force eviction
MaxInMemoryBuffer = 16,
PublishBatchSize = 4
});
var buffer = new RuntimeEventBuffer(options, TimeProvider.System, NullLogger<RuntimeEventBuffer>.Instance);
for (var i = 0; i < 5; i++)
{
var envelope = CreateEnvelope($"evt-{i}", annotationSize: 2048);
await buffer.WriteBatchAsync(new[] { envelope }, CancellationToken.None);
}
// Rehydrate to read what remained after capacity enforcement.
var restored = new RuntimeEventBuffer(options, TimeProvider.System, NullLogger<RuntimeEventBuffer>.Instance);
var enumerator = restored.ReadAllAsync(CancellationToken.None).GetAsyncEnumerator();
var ids = new List<string>();
while (true)
{
RuntimeEventBufferItem item;
try
{
var hasNext = await enumerator.MoveNextAsync().AsTask().WaitAsync(TimeSpan.FromMilliseconds(200));
if (!hasNext)
{
break;
}
item = enumerator.Current;
}
catch (TimeoutException)
{
break;
}
ids.Add(item.Envelope.Event.EventId);
await item.CompleteAsync();
}
// Oldest events should have been dropped; ensure fewer than written remain.
var totalBytes = Directory.GetFiles(bufferPath)
.Select(path => new FileInfo(path).Length)
.Sum();
Assert.True(totalBytes <= options.Value.MaxDiskBufferBytes, "Runtime event buffer exceeded configured capacity.");
Assert.True(ids.Count > 0, "Expected at least one runtime event to remain buffered.");
Assert.True(ids.Contains("evt-4"), "Most recent event should remain in buffer.");
}
private static RuntimeEventEnvelope CreateEnvelope(string id, int annotationSize = 0)
{
var annotations = annotationSize > 0
? new Dictionary<string, string> { ["blob"] = new string('x', annotationSize) }
: null;
var runtimeEvent = new RuntimeEvent
{
EventId = id,
When = DateTimeOffset.UtcNow,
Kind = RuntimeEventKind.ContainerStart,
Tenant = "tenant-a",
Node = "node-1",
Runtime = new RuntimeEngine
{
Engine = "containerd",
Version = "1.7.0"
},
Workload = new RuntimeWorkload
{
Platform = "kubernetes",
Namespace = "default",
Pod = "pod-1",
Container = "app",
ContainerId = "containerd://abc",
ImageRef = "ghcr.io/example/app@sha256:deadbeef"
},
Annotations = annotations
};
return RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent);
}
private static async Task<RuntimeEventBufferItem> ReadNextAsync(IAsyncEnumerator<RuntimeEventBufferItem> enumerator)
{
try
{
var hasNext = await enumerator.MoveNextAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(1));
Assert.True(hasNext, "Expected runtime event to be available in buffer.");
}
catch (TimeoutException)
{
Assert.Fail("Timed out waiting for runtime event from buffer.");
}
return enumerator.Current;
}
private sealed class TempDirectory : IDisposable
{
public TempDirectory()
{
RootPath = Path.Combine(Path.GetTempPath(), "observer-buffer-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(RootPath);
}
public string RootPath { get; }
public string CreateSubdirectory(string name)
{
var path = Path.Combine(RootPath, name);
Directory.CreateDirectory(path);
return path;
}
public void Dispose()
{
try
{
if (Directory.Exists(RootPath))
{
Directory.Delete(RootPath, recursive: true);
}
}
catch
{
}
}
}
}

View File

@@ -0,0 +1,156 @@
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
using StellaOps.Zastava.Observer.Runtime;
using StellaOps.Zastava.Observer.Tests.TestSupport;
using Xunit;
namespace StellaOps.Zastava.Observer.Tests.Runtime;
public sealed class RuntimeProcessCollectorTests
{
[Fact]
public async Task CollectAsync_ParsesCmdlineAndLibraries()
{
using var temp = new TempDirectory();
var procRoot = temp.CreateSubdirectory("proc");
var pidDir = Path.Combine(procRoot, "1234");
Directory.CreateDirectory(pidDir);
var cmdlineContent = Encoding.UTF8.GetBytes("/bin/bash\0-c\0python /app/server.py\0");
await File.WriteAllBytesAsync(Path.Combine(pidDir, "cmdline"), cmdlineContent);
var libPath = Path.Combine(temp.RootPath, "libs", "libexample.so");
Directory.CreateDirectory(Path.GetDirectoryName(libPath)!);
await File.WriteAllTextAsync(libPath, "library-bytes");
var buildIdBytes = Enumerable.Range(0, 20).Select(static index => (byte)(index + 1)).ToArray();
var exePath = Path.Combine(pidDir, "exe");
ElfTestFileBuilder.CreateElfWithBuildId(exePath, buildIdBytes);
var mapsLine = $"7f6d8c900000-7f6d8ca00000 r-xp 00000000 00:00 0 {libPath}";
await File.WriteAllTextAsync(Path.Combine(pidDir, "maps"), mapsLine + Environment.NewLine);
var options = Options.Create(new ZastavaObserverOptions
{
ProcRootPath = procRoot,
MaxTrackedLibraries = 8,
MaxEntrypointArguments = 16,
MaxLibraryBytes = 1024 * 1024
});
var collector = new RuntimeProcessCollector(options, NullLogger<RuntimeProcessCollector>.Instance);
var container = new CriContainerInfo(
Id: "container-1",
PodSandboxId: "sandbox",
Name: "example",
Attempt: 1,
Image: "ghcr.io/example/app:1.0",
ImageRef: "ghcr.io/example/app@sha256:deadbeef",
Labels: new Dictionary<string, string>(StringComparer.Ordinal),
Annotations: new Dictionary<string, string>(StringComparer.Ordinal),
CreatedAt: DateTimeOffset.UtcNow,
StartedAt: DateTimeOffset.UtcNow,
FinishedAt: null,
ExitCode: null,
Reason: null,
Message: null,
Pid: 1234);
var capture = await collector.CollectAsync(container, CancellationToken.None);
Assert.NotNull(capture);
Assert.NotNull(capture!.Process);
Assert.Contains("/bin/bash", capture.Process.Entrypoint);
Assert.Contains(capture.Process.EntryTrace, trace => trace.Op == "shell" && trace.Target == "python /app/server.py");
Assert.Contains(capture.Process.EntryTrace, trace => trace.Op == "python" && trace.Target == "/app/server.py");
Assert.NotEmpty(capture.Libraries);
var expectedHash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes("library-bytes"))).ToLowerInvariant();
Assert.Contains(capture.Libraries, lib => lib.Path == libPath && lib.Sha256 == expectedHash);
Assert.Contains(capture.Evidence, item => item.Signal == "procfs.maps" && item.Value == $"{libPath}@0x7f6d8c900000");
Assert.Contains(capture.Evidence, item => item.Signal == "procfs.maps.count" && item.Value == "1");
Assert.Contains(capture.Evidence, item => item.Signal == "procfs.cmdline");
Assert.Equal(Convert.ToHexString(buildIdBytes).ToLowerInvariant(), capture.Process.BuildId);
Assert.Contains(capture.Evidence, item => item.Signal == "procfs.buildId");
}
[Fact]
public async Task CollectAsync_NodeEntrypointProducesTrace()
{
using var temp = new TempDirectory();
var procRoot = temp.CreateSubdirectory("proc");
var pidDir = Path.Combine(procRoot, "4321");
Directory.CreateDirectory(pidDir);
await File.WriteAllBytesAsync(Path.Combine(pidDir, "cmdline"), Encoding.UTF8.GetBytes("/usr/bin/node\0/app/index.js\0"));
await File.WriteAllTextAsync(Path.Combine(pidDir, "maps"), string.Empty);
var options = Options.Create(new ZastavaObserverOptions
{
ProcRootPath = procRoot,
MaxTrackedLibraries = 8,
MaxEntrypointArguments = 16
});
var collector = new RuntimeProcessCollector(options, NullLogger<RuntimeProcessCollector>.Instance);
var container = new CriContainerInfo(
Id: "container-node",
PodSandboxId: "sandbox-node",
Name: "node-app",
Attempt: 1,
Image: "ghcr.io/example/node:1.0",
ImageRef: "ghcr.io/example/node@sha256:feedface",
Labels: new Dictionary<string, string>(StringComparer.Ordinal),
Annotations: new Dictionary<string, string>(StringComparer.Ordinal),
CreatedAt: DateTimeOffset.UtcNow,
StartedAt: DateTimeOffset.UtcNow,
FinishedAt: null,
ExitCode: null,
Reason: null,
Message: null,
Pid: 4321);
var capture = await collector.CollectAsync(container, CancellationToken.None);
Assert.NotNull(capture);
Assert.NotNull(capture!.Process);
Assert.Contains("/usr/bin/node", capture.Process.Entrypoint);
Assert.Contains(capture.Process.EntryTrace, trace => trace.Op == "node" && trace.Target == "/app/index.js");
}
private sealed class TempDirectory : IDisposable
{
public TempDirectory()
{
RootPath = Path.Combine(Path.GetTempPath(), "observer-tests", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(RootPath);
}
public string RootPath { get; }
public string CreateSubdirectory(string name)
{
var path = Path.Combine(RootPath, name);
Directory.CreateDirectory(path);
return path;
}
public void Dispose()
{
try
{
if (Directory.Exists(RootPath))
{
Directory.Delete(RootPath, recursive: true);
}
}
catch
{
}
}
}
}

View File

@@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Zastava.Observer\StellaOps.Zastava.Observer.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,73 @@
using System.Buffers.Binary;
using System.IO;
namespace StellaOps.Zastava.Observer.Tests.TestSupport;
internal static class ElfTestFileBuilder
{
private const int HeaderSize = 64;
private const int ProgramHeaderSize = 56;
private const uint ProgramHeaderTypeNote = 4;
private const uint NoteTypeGnuBuildId = 3;
public static void CreateElfWithBuildId(string path, ReadOnlySpan<byte> buildId)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("Path cannot be null or whitespace.", nameof(path));
}
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
}
var nameBytes = new byte[] { (byte)'G', (byte)'N', (byte)'U', 0 };
var alignedNameSize = Align(nameBytes.Length);
var alignedDescSize = Align(buildId.Length);
var noteSize = 12 + alignedNameSize + alignedDescSize;
var noteOffset = HeaderSize + ProgramHeaderSize;
var totalSize = noteOffset + noteSize;
var buffer = new byte[totalSize];
var span = buffer.AsSpan();
// ELF ident
span[0] = 0x7F;
span[1] = (byte)'E';
span[2] = (byte)'L';
span[3] = (byte)'F';
span[4] = 2; // 64-bit
span[5] = 1; // little-endian
span[6] = 1; // version
BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(16, 2), 2); // e_type
BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(18, 2), 0x3E); // e_machine (x86-64)
BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(20, 4), 1); // e_version
BinaryPrimitives.WriteUInt64LittleEndian(span.Slice(32, 8), HeaderSize); // e_phoff
BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(52, 2), HeaderSize); // e_ehsize
BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(54, 2), ProgramHeaderSize); // e_phentsize
BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(56, 2), 1); // e_phnum
var programHeader = span.Slice(HeaderSize, ProgramHeaderSize);
BinaryPrimitives.WriteUInt32LittleEndian(programHeader.Slice(0, 4), ProgramHeaderTypeNote);
BinaryPrimitives.WriteUInt64LittleEndian(programHeader.Slice(8, 8), (ulong)noteOffset);
BinaryPrimitives.WriteUInt64LittleEndian(programHeader.Slice(32, 8), (ulong)noteSize);
BinaryPrimitives.WriteUInt64LittleEndian(programHeader.Slice(40, 8), (ulong)noteSize);
BinaryPrimitives.WriteUInt64LittleEndian(programHeader.Slice(48, 8), 4);
var note = span.Slice(noteOffset, noteSize);
BinaryPrimitives.WriteUInt32LittleEndian(note.Slice(0, 4), (uint)nameBytes.Length);
BinaryPrimitives.WriteUInt32LittleEndian(note.Slice(4, 4), (uint)buildId.Length);
BinaryPrimitives.WriteUInt32LittleEndian(note.Slice(8, 4), NoteTypeGnuBuildId);
nameBytes.CopyTo(note.Slice(12, nameBytes.Length));
var descriptorStart = 12 + alignedNameSize;
buildId.CopyTo(note.Slice(descriptorStart, buildId.Length));
File.WriteAllBytes(path, buffer);
}
private static int Align(int value) => (value + 3) & ~3;
}

View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Zastava.Observer.Backend;
internal interface IRuntimePolicyClient
{
Task<RuntimePolicyResponse> EvaluateAsync(RuntimePolicyRequest request, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,237 @@
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Configuration;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Core.Diagnostics;
using StellaOps.Zastava.Core.Security;
using StellaOps.Zastava.Core.Serialization;
using StellaOps.Zastava.Observer.Configuration;
namespace StellaOps.Zastava.Observer.Backend;
internal interface IRuntimeEventsClient
{
Task<RuntimeEventPublishResult> PublishAsync(RuntimeEventsIngestRequest request, CancellationToken cancellationToken);
}
internal sealed class RuntimeEventsClient : IRuntimeEventsClient
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
static RuntimeEventsClient()
{
SerializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false));
}
private readonly HttpClient httpClient;
private readonly IZastavaAuthorityTokenProvider authorityTokenProvider;
private readonly IOptionsMonitor<ZastavaRuntimeOptions> runtimeOptions;
private readonly IOptionsMonitor<ZastavaObserverOptions> observerOptions;
private readonly IZastavaRuntimeMetrics runtimeMetrics;
private readonly ILogger<RuntimeEventsClient> logger;
public RuntimeEventsClient(
HttpClient httpClient,
IZastavaAuthorityTokenProvider authorityTokenProvider,
IOptionsMonitor<ZastavaRuntimeOptions> runtimeOptions,
IOptionsMonitor<ZastavaObserverOptions> observerOptions,
IZastavaRuntimeMetrics runtimeMetrics,
ILogger<RuntimeEventsClient> logger)
{
this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
this.authorityTokenProvider = authorityTokenProvider ?? throw new ArgumentNullException(nameof(authorityTokenProvider));
this.runtimeOptions = runtimeOptions ?? throw new ArgumentNullException(nameof(runtimeOptions));
this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions));
this.runtimeMetrics = runtimeMetrics ?? throw new ArgumentNullException(nameof(runtimeMetrics));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RuntimeEventPublishResult> PublishAsync(RuntimeEventsIngestRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
if (request.Events.Count == 0)
{
return RuntimeEventPublishResult.Empty;
}
var runtime = runtimeOptions.CurrentValue;
var authority = runtime.Authority;
var audience = authority.Audience.FirstOrDefault() ?? "scanner";
var scopes = authority.Scopes ?? Array.Empty<string>();
var token = await authorityTokenProvider.GetAsync(audience, scopes, cancellationToken).ConfigureAwait(false);
var backend = observerOptions.CurrentValue.Backend;
var requestPath = backend.EventsPath;
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, requestPath);
var payload = ZastavaCanonicalJsonSerializer.SerializeToUtf8Bytes(request);
httpRequest.Content = new ByteArrayContent(payload);
httpRequest.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpRequest.Headers.Authorization = CreateAuthorizationHeader(token);
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
try
{
using var response = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
stopwatch.Stop();
RecordLatency(stopwatch.Elapsed.TotalMilliseconds, success: response.IsSuccessStatusCode);
if (response.IsSuccessStatusCode)
{
var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
RuntimeEventsIngestResponse? parsed = null;
if (!string.IsNullOrWhiteSpace(body))
{
parsed = JsonSerializer.Deserialize<RuntimeEventsIngestResponse>(body, SerializerOptions);
}
var accepted = parsed?.Accepted ?? request.Events.Count;
var duplicates = parsed?.Duplicates ?? 0;
logger.LogDebug("Published runtime events batch (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}).",
request.BatchId,
accepted,
duplicates);
return RuntimeEventPublishResult.Successful(accepted, duplicates);
}
if (response.StatusCode == HttpStatusCode.TooManyRequests)
{
var retryAfter = ParseRetryAfter(response.Headers.RetryAfter) ?? TimeSpan.FromSeconds(5);
logger.LogWarning("Runtime events publish rate limited (batchId={BatchId}, retryAfter={RetryAfter}).", request.BatchId, retryAfter);
return RuntimeEventPublishResult.FromRateLimit(retryAfter);
}
var errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
logger.LogWarning("Runtime events publish failed with status {Status} (batchId={BatchId}): {Payload}",
(int)response.StatusCode,
request.BatchId,
Truncate(errorBody));
throw new RuntimeEventsException($"Runtime events publish failed with status {(int)response.StatusCode}", response.StatusCode);
}
catch (RuntimeEventsException)
{
throw;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
stopwatch.Stop();
RecordLatency(stopwatch.Elapsed.TotalMilliseconds, success: false);
logger.LogWarning(ex, "Runtime events publish encountered an exception (batchId={BatchId}).", request.BatchId);
throw new RuntimeEventsException("Runtime events publish failed due to network error.", HttpStatusCode.ServiceUnavailable, ex);
}
}
private AuthenticationHeaderValue CreateAuthorizationHeader(ZastavaOperationalToken token)
{
var scheme = string.Equals(token.TokenType, "dpop", StringComparison.OrdinalIgnoreCase)
? "DPoP"
: token.TokenType;
return new AuthenticationHeaderValue(scheme, token.AccessToken);
}
private void RecordLatency(double elapsedMs, bool success)
{
var tags = runtimeMetrics.DefaultTags
.Concat(new[]
{
new KeyValuePair<string, object?>("endpoint", "runtime-events"),
new KeyValuePair<string, object?>("success", success ? "true" : "false")
})
.ToArray();
runtimeMetrics.BackendLatencyMs.Record(elapsedMs, tags);
}
private static TimeSpan? ParseRetryAfter(RetryConditionHeaderValue? retryAfter)
{
if (retryAfter is null)
{
return null;
}
if (retryAfter.Delta.HasValue)
{
return retryAfter.Delta.Value;
}
if (retryAfter.Date.HasValue)
{
var delta = retryAfter.Date.Value.UtcDateTime - DateTime.UtcNow;
return delta > TimeSpan.Zero ? delta : TimeSpan.Zero;
}
return null;
}
private static string Truncate(string? value, int maxLength = 512)
{
if (string.IsNullOrEmpty(value))
{
return string.Empty;
}
return value.Length <= maxLength ? value : value[..maxLength] + "…";
}
}
internal sealed record RuntimeEventsIngestRequest
{
[JsonPropertyName("batchId")]
public string? BatchId { get; init; }
[JsonPropertyName("events")]
public IReadOnlyList<RuntimeEventEnvelope> Events { get; init; } = Array.Empty<RuntimeEventEnvelope>();
}
internal sealed record RuntimeEventsIngestResponse
{
[JsonPropertyName("accepted")]
public int Accepted { get; init; }
[JsonPropertyName("duplicates")]
public int Duplicates { get; init; }
}
internal readonly record struct RuntimeEventPublishResult(
bool Success,
bool RateLimited,
TimeSpan RetryAfter,
int Accepted,
int Duplicates)
{
public static RuntimeEventPublishResult Empty => new(true, false, TimeSpan.Zero, 0, 0);
public static RuntimeEventPublishResult Successful(int accepted, int duplicates)
=> new(true, false, TimeSpan.Zero, accepted, duplicates);
public static RuntimeEventPublishResult FromRateLimit(TimeSpan retryAfter)
=> new(false, true, retryAfter, 0, 0);
}
internal sealed class RuntimeEventsException : Exception
{
public RuntimeEventsException(string message, HttpStatusCode statusCode, Exception? innerException = null)
: base(message, innerException)
{
StatusCode = statusCode;
}
public HttpStatusCode StatusCode { get; }
}

View File

@@ -0,0 +1,128 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Configuration;
using StellaOps.Zastava.Core.Diagnostics;
using StellaOps.Zastava.Core.Security;
using StellaOps.Zastava.Observer.Configuration;
namespace StellaOps.Zastava.Observer.Backend;
internal sealed class RuntimePolicyClient : IRuntimePolicyClient
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
static RuntimePolicyClient()
{
SerializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false));
}
private readonly HttpClient httpClient;
private readonly IZastavaAuthorityTokenProvider authorityTokenProvider;
private readonly IOptionsMonitor<ZastavaRuntimeOptions> runtimeOptions;
private readonly IOptionsMonitor<ZastavaObserverOptions> observerOptions;
private readonly IZastavaRuntimeMetrics runtimeMetrics;
private readonly ILogger<RuntimePolicyClient> logger;
public RuntimePolicyClient(
HttpClient httpClient,
IZastavaAuthorityTokenProvider authorityTokenProvider,
IOptionsMonitor<ZastavaRuntimeOptions> runtimeOptions,
IOptionsMonitor<ZastavaObserverOptions> observerOptions,
IZastavaRuntimeMetrics runtimeMetrics,
ILogger<RuntimePolicyClient> logger)
{
this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
this.authorityTokenProvider = authorityTokenProvider ?? throw new ArgumentNullException(nameof(authorityTokenProvider));
this.runtimeOptions = runtimeOptions ?? throw new ArgumentNullException(nameof(runtimeOptions));
this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions));
this.runtimeMetrics = runtimeMetrics ?? throw new ArgumentNullException(nameof(runtimeMetrics));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RuntimePolicyResponse> EvaluateAsync(RuntimePolicyRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var runtime = runtimeOptions.CurrentValue;
var authority = runtime.Authority;
var audience = authority.Audience.FirstOrDefault() ?? "scanner";
var token = await authorityTokenProvider
.GetAsync(audience, authority.Scopes ?? Array.Empty<string>(), cancellationToken)
.ConfigureAwait(false);
var backend = observerOptions.CurrentValue.Backend;
EnsureBackendGuardrails(backend);
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, backend.PolicyPath)
{
Content = new StringContent(JsonSerializer.Serialize(request, SerializerOptions), Encoding.UTF8, "application/json")
};
httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpRequest.Headers.Authorization = CreateAuthorizationHeader(token);
var stopwatch = Stopwatch.StartNew();
try
{
using var response = await httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false);
var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
logger.LogWarning("Runtime policy call returned {StatusCode}: {Payload}", (int)response.StatusCode, payload);
throw new RuntimePolicyException($"Runtime policy call failed with status {(int)response.StatusCode}", response.StatusCode);
}
var result = JsonSerializer.Deserialize<RuntimePolicyResponse>(payload, SerializerOptions);
if (result is null)
{
throw new RuntimePolicyException("Runtime policy response payload was empty or invalid.", response.StatusCode);
}
return result;
}
finally
{
stopwatch.Stop();
RecordLatency(stopwatch.Elapsed.TotalMilliseconds);
}
}
private AuthenticationHeaderValue CreateAuthorizationHeader(ZastavaOperationalToken token)
{
var scheme = string.Equals(token.TokenType, "dpop", StringComparison.OrdinalIgnoreCase) ? "DPoP" : token.TokenType;
return new AuthenticationHeaderValue(scheme, token.AccessToken);
}
private void RecordLatency(double elapsedMs)
{
var tags = runtimeMetrics.DefaultTags
.Concat(new[] { new KeyValuePair<string, object?>("endpoint", "policy") })
.ToArray();
runtimeMetrics.BackendLatencyMs.Record(elapsedMs, tags);
}
private static void EnsureBackendGuardrails(ZastavaObserverBackendOptions backend)
{
if (!backend.AllowInsecureHttp && !string.Equals(backend.BaseAddress.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Observer backend baseAddress must use HTTPS unless allowInsecureHttp is true.");
}
}
}

View File

@@ -0,0 +1,73 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
using StellaOps.Zastava.Core.Contracts;
namespace StellaOps.Zastava.Observer.Backend;
internal sealed record RuntimePolicyRequest
{
[JsonPropertyName("namespace")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Namespace { get; init; }
[JsonPropertyName("labels")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IReadOnlyDictionary<string, string>? Labels { get; init; }
[JsonPropertyName("images")]
public required IReadOnlyList<string> Images { get; init; }
}
internal sealed record RuntimePolicyResponse
{
[JsonPropertyName("ttlSeconds")]
public int TtlSeconds { get; init; }
[JsonPropertyName("expiresAtUtc")]
public DateTimeOffset ExpiresAtUtc { get; init; }
[JsonPropertyName("policyRevision")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? PolicyRevision { get; init; }
[JsonPropertyName("results")]
public IReadOnlyDictionary<string, RuntimePolicyImageResult> Results { get; init; } = new Dictionary<string, RuntimePolicyImageResult>(StringComparer.Ordinal);
}
internal sealed record RuntimePolicyImageResult
{
[JsonPropertyName("policyVerdict")]
public PolicyVerdict PolicyVerdict { get; init; } = PolicyVerdict.Error;
[JsonPropertyName("signed")]
public bool Signed { get; init; }
[JsonPropertyName("hasSbomReferrers")]
public bool HasSbomReferrers { get; init; }
[JsonPropertyName("hasSbom")]
public bool HasSbomLegacy { get; init; }
[JsonPropertyName("reasons")]
public IReadOnlyList<string> Reasons { get; init; } = Array.Empty<string>();
[JsonPropertyName("rekor")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public RuntimePolicyRekorResult? Rekor { get; init; }
}
internal sealed record RuntimePolicyRekorResult
{
[JsonPropertyName("uuid")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Uuid { get; init; }
[JsonPropertyName("url")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Url { get; init; }
[JsonPropertyName("verified")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public bool? Verified { get; init; }
}

View File

@@ -0,0 +1,21 @@
using System;
using System.Net;
namespace StellaOps.Zastava.Observer.Backend;
internal sealed class RuntimePolicyException : Exception
{
public RuntimePolicyException(string message, HttpStatusCode statusCode)
: base(message)
{
StatusCode = statusCode;
}
public RuntimePolicyException(string message, HttpStatusCode statusCode, Exception innerException)
: base(message, innerException)
{
StatusCode = statusCode;
}
public HttpStatusCode StatusCode { get; }
}

View File

@@ -1,4 +1,5 @@
using System.ComponentModel.DataAnnotations;
using System.IO;
namespace StellaOps.Zastava.Observer.Configuration;
@@ -38,6 +39,24 @@ public sealed class ZastavaObserverOptions
[Range(1, 512)]
public int PublishBatchSize { get; set; } = 32;
/// <summary>
/// Maximum interval (seconds) that events may remain buffered before forcing a publish.
/// </summary>
[Range(typeof(double), "0.1", "30")]
public double PublishFlushIntervalSeconds { get; set; } = 2;
/// <summary>
/// Directory used for disk-backed runtime event buffering.
/// </summary>
[Required(AllowEmptyStrings = false)]
public string EventBufferPath { get; set; } = Path.Combine(Path.GetTempPath(), "zastava-observer", "runtime-events");
/// <summary>
/// Maximum on-disk bytes retained for buffered runtime events.
/// </summary>
[Range(typeof(long), "1048576", "1073741824")]
public long MaxDiskBufferBytes { get; set; } = 64 * 1024 * 1024; // 64 MiB
/// <summary>
/// Connectivity/backoff settings applied when CRI endpoints fail temporarily.
/// </summary>
@@ -58,6 +77,101 @@ public sealed class ZastavaObserverOptions
Enabled = true
}
};
/// <summary>
/// Scanner backend configuration for posture checks and event ingestion.
/// </summary>
[Required]
public ZastavaObserverBackendOptions Backend { get; set; } = new();
/// <summary>
/// Posture-specific configuration values.
/// </summary>
[Required]
public ZastavaObserverPostureOptions Posture { get; set; } = new();
/// <summary>
/// Root path for accessing host process information (defaults to /host/proc).
/// </summary>
[Required(AllowEmptyStrings = false)]
public string ProcRootPath { get; set; } = "/host/proc";
/// <summary>
/// Maximum number of loaded libraries captured per process.
/// </summary>
[Range(8, 4096)]
public int MaxTrackedLibraries { get; set; } = 256;
/// <summary>
/// Maximum size (in bytes) of a library file to hash when collecting loaded libraries.
/// </summary>
[Range(typeof(long), "1024", "1073741824")]
public long MaxLibraryBytes { get; set; } = 33554432; // 32 MiB
/// <summary>
/// Maximum cumulative bytes hashed across libraries for a single process capture.
/// </summary>
[Range(typeof(long), "1024", "2147483647")]
public long MaxLibraryHashBytes { get; set; } = 64_000_000; // ~61 MiB budget
/// <summary>
/// Maximum number of entrypoint arguments captured for reporting.
/// </summary>
[Range(1, 128)]
public int MaxEntrypointArguments { get; set; } = 32;
}
public sealed class ZastavaObserverBackendOptions
{
/// <summary>
/// Base address for Scanner WebService runtime APIs.
/// </summary>
[Required]
public Uri BaseAddress { get; init; } = new("https://scanner.internal");
/// <summary>
/// Runtime policy endpoint path.
/// </summary>
[Required(AllowEmptyStrings = false)]
public string PolicyPath { get; init; } = "/api/v1/scanner/policy/runtime";
/// <summary>
/// Runtime events ingestion endpoint path.
/// </summary>
[Required(AllowEmptyStrings = false)]
public string EventsPath { get; init; } = "/api/v1/runtime/events";
/// <summary>
/// Request timeout for backend calls in seconds.
/// </summary>
[Range(typeof(double), "1", "120")]
public double RequestTimeoutSeconds { get; init; } = 5;
/// <summary>
/// Allows plain HTTP endpoints when true (default false for safety).
/// </summary>
public bool AllowInsecureHttp { get; init; }
}
public sealed class ZastavaObserverPostureOptions
{
/// <summary>
/// Path where posture cache entries are persisted across restarts.
/// </summary>
[Required(AllowEmptyStrings = false)]
public string CachePath { get; init; } = Path.Combine(Path.GetTempPath(), "zastava-observer", "posture-cache.json");
/// <summary>
/// Fallback TTL (seconds) applied when backend omits an explicit expiry.
/// </summary>
[Range(30, 86400)]
public int FallbackTtlSeconds { get; init; } = 300;
/// <summary>
/// Threshold (seconds) after expiration where stale cache usage triggers warnings.
/// </summary>
[Range(30, 86400)]
public int StaleWarningThresholdSeconds { get; init; } = 900;
}
public sealed class ObserverBackoffOptions

View File

@@ -0,0 +1,7 @@
namespace StellaOps.Zastava.Observer.ContainerRuntime;
internal sealed class ContainerStateTrackerFactory
{
public ContainerStateTracker Create()
=> new();
}

View File

@@ -24,7 +24,8 @@ internal static class CriConversions
FinishedAt: null,
ExitCode: null,
Reason: null,
Message: null);
Message: null,
Pid: null);
}
public static CriContainerInfo MergeStatus(CriContainerInfo baseline, ContainerStatus? status)
@@ -47,8 +48,9 @@ internal static class CriConversions
ExitCode = status.ExitCode != 0 ? status.ExitCode : baseline.ExitCode,
Reason = string.IsNullOrWhiteSpace(status.Reason) ? baseline.Reason : status.Reason,
Message = string.IsNullOrWhiteSpace(status.Message) ? baseline.Message : status.Message,
Image: status.Image?.Image ?? baseline.Image,
ImageRef: string.IsNullOrWhiteSpace(status.ImageRef) ? baseline.ImageRef : status.ImageRef,
Pid = baseline.Pid,
Image = status.Image?.Image ?? baseline.Image,
ImageRef = string.IsNullOrWhiteSpace(status.ImageRef) ? baseline.ImageRef : status.ImageRef,
Labels = labels,
Annotations = annotations
};

View File

@@ -21,7 +21,8 @@ internal sealed record CriContainerInfo(
DateTimeOffset? FinishedAt,
int? ExitCode,
string? Reason,
string? Message);
string? Message,
int? Pid);
internal static class CriLabelKeys
{

View File

@@ -1,6 +1,7 @@
using System.IO;
using System.Net.Sockets;
using System.Linq;
using System.Net.Sockets;
using System.Text.Json;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Logging;
@@ -92,7 +93,7 @@ internal sealed class CriRuntimeClient : ICriRuntimeClient
var response = await client.ContainerStatusAsync(new ContainerStatusRequest
{
ContainerId = containerId,
Verbose = false
Verbose = true
}, cancellationToken: cancellationToken).ConfigureAwait(false);
if (response.Status is null)
@@ -112,7 +113,14 @@ internal sealed class CriRuntimeClient : ICriRuntimeClient
CreatedAt = response.Status.CreatedAt
});
return CriConversions.MergeStatus(baseline, response.Status);
var merged = CriConversions.MergeStatus(baseline, response.Status);
if (response.Info is { Count: > 0 } && TryExtractPid(response.Info, out var pid))
{
merged = merged with { Pid = pid };
}
return merged;
}
catch (RpcException ex) when (ex.StatusCode is StatusCode.NotFound or StatusCode.DeadlineExceeded)
{
@@ -121,16 +129,49 @@ internal sealed class CriRuntimeClient : ICriRuntimeClient
}
}
public async ValueTask DisposeAsync()
private static bool TryExtractPid(IDictionary<string, string> info, out int pid)
{
if (info.TryGetValue("pid", out var value) && int.TryParse(value, out pid))
{
return true;
}
foreach (var entry in info.Values)
{
if (string.IsNullOrWhiteSpace(entry))
{
continue;
}
try
{
using var document = JsonDocument.Parse(entry);
if (document.RootElement.TryGetProperty("pid", out var pidElement) && pidElement.TryGetInt32(out pid))
{
return true;
}
}
catch (JsonException)
{
}
}
pid = default;
return false;
}
public ValueTask DisposeAsync()
{
try
{
await channel.DisposeAsync().ConfigureAwait(false);
channel.Dispose();
}
catch (InvalidOperationException)
{
// Channel already disposed.
}
return ValueTask.CompletedTask;
}
private static void EnsureHttp2Switch()
@@ -161,7 +202,7 @@ internal sealed class CriRuntimeClient : ICriRuntimeClient
EnableMultipleHttp2Connections = true
};
if (endpoint.ConnectTimeout is { } timeout and > TimeSpan.Zero)
if (endpoint.ConnectTimeout is { } timeout && timeout > TimeSpan.Zero)
{
handler.ConnectTimeout = timeout;
}

View File

@@ -0,0 +1,103 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Configuration;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
using StellaOps.Zastava.Observer.ContainerRuntime;
using StellaOps.Zastava.Observer.Posture;
using StellaOps.Zastava.Observer.Runtime;
using StellaOps.Zastava.Observer.Worker;
using StellaOps.Zastava.Observer.Backend;
namespace Microsoft.Extensions.DependencyInjection;
public static class ObserverServiceCollectionExtensions
{
public static IServiceCollection AddZastavaObserver(this IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddZastavaRuntimeCore(configuration, componentName: "observer");
services.AddOptions<ZastavaObserverOptions>()
.Bind(configuration.GetSection(ZastavaObserverOptions.SectionName))
.ValidateDataAnnotations()
.PostConfigure(options =>
{
if (options.Backoff.Initial <= TimeSpan.Zero)
{
options.Backoff.Initial = TimeSpan.FromSeconds(1);
}
if (options.Backoff.Max < options.Backoff.Initial)
{
options.Backoff.Max = options.Backoff.Initial;
}
if (!options.Backend.AllowInsecureHttp && !string.Equals(options.Backend.BaseAddress.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Observer backend baseAddress must use HTTPS unless allowInsecureHttp is explicitly enabled.");
}
if (!options.Backend.PolicyPath.StartsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("Observer backend policyPath must be absolute (start with '/').");
}
if (!options.Backend.EventsPath.StartsWith("/", StringComparison.Ordinal))
{
throw new InvalidOperationException("Observer backend eventsPath must be absolute (start with '/').");
}
})
.ValidateOnStart();
services.TryAddSingleton(TimeProvider.System);
services.TryAddSingleton<ICriRuntimeClientFactory, CriRuntimeClientFactory>();
services.TryAddSingleton<IRuntimeEventBuffer, RuntimeEventBuffer>();
services.TryAddSingleton<IRuntimeProcessCollector, RuntimeProcessCollector>();
services.TryAddSingleton<IRuntimePostureCache, RuntimePostureCache>();
services.TryAddSingleton<IRuntimePostureEvaluator, RuntimePostureEvaluator>();
services.TryAddSingleton<ContainerStateTrackerFactory>();
services.TryAddSingleton<ContainerRuntimePoller>();
services.AddHttpClient<IRuntimePolicyClient, RuntimePolicyClient>()
.ConfigureHttpClient((provider, client) =>
{
var optionsMonitor = provider.GetRequiredService<IOptionsMonitor<ZastavaObserverOptions>>();
var backend = optionsMonitor.CurrentValue.Backend;
client.BaseAddress = backend.BaseAddress;
client.Timeout = TimeSpan.FromSeconds(Math.Clamp(backend.RequestTimeoutSeconds, 1, 120));
});
services.AddHttpClient<IRuntimeEventsClient, RuntimeEventsClient>()
.ConfigureHttpClient((provider, client) =>
{
var optionsMonitor = provider.GetRequiredService<IOptionsMonitor<ZastavaObserverOptions>>();
var backend = optionsMonitor.CurrentValue.Backend;
client.BaseAddress = backend.BaseAddress;
client.Timeout = TimeSpan.FromSeconds(Math.Clamp(backend.RequestTimeoutSeconds, 1, 120));
});
services.TryAddEnumerable(ServiceDescriptor.Singleton<IPostConfigureOptions<ZastavaRuntimeOptions>, ObserverRuntimeOptionsPostConfigure>());
services.AddHostedService<ObserverBootstrapService>();
services.AddHostedService<ContainerLifecycleHostedService>();
services.AddHostedService<RuntimeEventDispatchService>();
return services;
}
}
internal sealed class ObserverRuntimeOptionsPostConfigure : IPostConfigureOptions<ZastavaRuntimeOptions>
{
public void PostConfigure(string? name, ZastavaRuntimeOptions options)
{
if (string.IsNullOrWhiteSpace(options.Component))
{
options.Component = "observer";
}
}
}

View File

@@ -0,0 +1,12 @@
using System;
using StellaOps.Zastava.Core.Contracts;
namespace StellaOps.Zastava.Observer.Posture;
internal interface IRuntimePostureCache
{
RuntimePostureCacheEntry? Get(string key);
void Set(string key, RuntimePosture posture, DateTimeOffset expiresAtUtc, DateTimeOffset storedAtUtc);
}

View File

@@ -0,0 +1,10 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
namespace StellaOps.Zastava.Observer.Posture;
internal interface IRuntimePostureEvaluator
{
Task<RuntimePostureEvaluationResult> EvaluateAsync(CriContainerInfo container, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,180 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Configuration;
namespace StellaOps.Zastava.Observer.Posture;
internal sealed class RuntimePostureCache : IRuntimePostureCache
{
private const int CurrentVersion = 1;
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
private readonly IOptionsMonitor<ZastavaObserverOptions> optionsMonitor;
private readonly ILogger<RuntimePostureCache> logger;
private readonly object entriesLock = new();
private readonly object fileLock = new();
private readonly Dictionary<string, RuntimePostureCacheEntry> entries = new(StringComparer.Ordinal);
public RuntimePostureCache(
IOptionsMonitor<ZastavaObserverOptions> optionsMonitor,
ILogger<RuntimePostureCache> logger)
{
this.optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
Load();
}
public RuntimePostureCacheEntry? Get(string key)
{
if (string.IsNullOrWhiteSpace(key))
{
return null;
}
lock (entriesLock)
{
return entries.TryGetValue(key, out var entry) ? entry : null;
}
}
public void Set(string key, RuntimePosture posture, DateTimeOffset expiresAtUtc, DateTimeOffset storedAtUtc)
{
if (string.IsNullOrWhiteSpace(key))
{
return;
}
ArgumentNullException.ThrowIfNull(posture);
var normalizedKey = key.Trim();
var entry = new RuntimePostureCacheEntry(posture, expiresAtUtc, storedAtUtc);
lock (entriesLock)
{
entries[normalizedKey] = entry;
}
Persist();
}
private void Load()
{
var path = GetCachePath();
if (!File.Exists(path))
{
return;
}
try
{
var json = File.ReadAllText(path);
var snapshot = JsonSerializer.Deserialize<CacheFileModel>(json, SerializerOptions);
if (snapshot?.Entries is null)
{
return;
}
lock (entriesLock)
{
entries.Clear();
foreach (var entry in snapshot.Entries)
{
if (string.IsNullOrWhiteSpace(entry.Key) || entry.Posture is null)
{
continue;
}
entries[entry.Key] = new RuntimePostureCacheEntry(
entry.Posture,
entry.ExpiresAtUtc,
entry.StoredAtUtc);
}
}
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to load runtime posture cache from {CachePath}; starting empty.", path);
}
}
private void Persist()
{
var path = GetCachePath();
var directory = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
}
CacheFileModel snapshot;
lock (entriesLock)
{
var ordered = entries
.OrderBy(pair => pair.Key, StringComparer.Ordinal)
.Select(pair => new CacheFileEntry
{
Key = pair.Key,
ExpiresAtUtc = pair.Value.ExpiresAtUtc,
StoredAtUtc = pair.Value.StoredAtUtc,
Posture = pair.Value.Posture
})
.ToList();
snapshot = new CacheFileModel
{
Version = CurrentVersion,
Entries = ordered
};
}
var json = JsonSerializer.Serialize(snapshot, SerializerOptions);
lock (fileLock)
{
var tempPath = path + ".tmp";
File.WriteAllText(tempPath, json);
File.Move(tempPath, path, overwrite: true);
}
}
private string GetCachePath()
{
return optionsMonitor.CurrentValue.Posture.CachePath;
}
private sealed record CacheFileModel
{
[JsonPropertyName("version")]
public int Version { get; init; }
[JsonPropertyName("entries")]
public List<CacheFileEntry> Entries { get; init; } = new();
}
private sealed record CacheFileEntry
{
[JsonPropertyName("key")]
public string Key { get; init; } = string.Empty;
[JsonPropertyName("expiresAtUtc")]
public DateTimeOffset ExpiresAtUtc { get; init; }
[JsonPropertyName("storedAtUtc")]
public DateTimeOffset StoredAtUtc { get; init; }
[JsonPropertyName("posture")]
public RuntimePosture? Posture { get; init; }
}
}

View File

@@ -0,0 +1,12 @@
using System;
using StellaOps.Zastava.Core.Contracts;
namespace StellaOps.Zastava.Observer.Posture;
internal sealed record RuntimePostureCacheEntry(RuntimePosture Posture, DateTimeOffset ExpiresAtUtc, DateTimeOffset StoredAtUtc)
{
public bool IsExpired(DateTimeOffset now) => now >= ExpiresAtUtc;
public bool IsStale(DateTimeOffset now, TimeSpan staleThreshold)
=> now - ExpiresAtUtc >= staleThreshold;
}

View File

@@ -0,0 +1,6 @@
using System.Collections.Generic;
using StellaOps.Zastava.Core.Contracts;
namespace StellaOps.Zastava.Observer.Posture;
internal sealed record RuntimePostureEvaluationResult(RuntimePosture? Posture, IReadOnlyList<RuntimeEvidence> Evidence);

View File

@@ -0,0 +1,188 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Backend;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
namespace StellaOps.Zastava.Observer.Posture;
internal sealed class RuntimePostureEvaluator : IRuntimePostureEvaluator
{
private readonly IRuntimePolicyClient policyClient;
private readonly IRuntimePostureCache cache;
private readonly IOptionsMonitor<ZastavaObserverOptions> optionsMonitor;
private readonly TimeProvider timeProvider;
private readonly ILogger<RuntimePostureEvaluator> logger;
public RuntimePostureEvaluator(
IRuntimePolicyClient policyClient,
IRuntimePostureCache cache,
IOptionsMonitor<ZastavaObserverOptions> optionsMonitor,
TimeProvider timeProvider,
ILogger<RuntimePostureEvaluator> logger)
{
this.policyClient = policyClient ?? throw new ArgumentNullException(nameof(policyClient));
this.cache = cache ?? throw new ArgumentNullException(nameof(cache));
this.optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
this.timeProvider = timeProvider ?? TimeProvider.System;
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RuntimePostureEvaluationResult> EvaluateAsync(CriContainerInfo container, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(container);
var evidence = new List<RuntimeEvidence>();
var now = timeProvider.GetUtcNow();
var cacheOptions = optionsMonitor.CurrentValue.Posture;
var fallbackTtl = TimeSpan.FromSeconds(Math.Clamp(cacheOptions.FallbackTtlSeconds, 30, 86400));
var staleThreshold = TimeSpan.FromSeconds(Math.Clamp(cacheOptions.StaleWarningThresholdSeconds, 30, 86400));
var imageKey = ResolveImageKey(container);
if (string.IsNullOrWhiteSpace(imageKey))
{
evidence.Add(new RuntimeEvidence
{
Signal = "runtime.posture.skipped",
Value = "no-image-ref"
});
return new RuntimePostureEvaluationResult(null, evidence);
}
var cached = cache.Get(imageKey);
if (cached is not null && !cached.IsExpired(now))
{
evidence.Add(new RuntimeEvidence
{
Signal = "runtime.posture.cache",
Value = "hit"
});
return new RuntimePostureEvaluationResult(cached.Posture, evidence);
}
try
{
var request = BuildRequest(container, imageKey);
var response = await policyClient.EvaluateAsync(request, cancellationToken).ConfigureAwait(false);
if (!response.Results.TryGetValue(imageKey, out var imageResult))
{
evidence.Add(new RuntimeEvidence
{
Signal = "runtime.posture.missing",
Value = "policy-empty"
});
return new RuntimePostureEvaluationResult(null, evidence);
}
var posture = MapPosture(imageResult);
var expiresAt = response.ExpiresAtUtc != default
? response.ExpiresAtUtc
: now.AddSeconds(response.TtlSeconds > 0 ? response.TtlSeconds : fallbackTtl.TotalSeconds);
cache.Set(imageKey, posture, expiresAt, now);
evidence.Add(new RuntimeEvidence
{
Signal = "runtime.posture.source",
Value = "backend"
});
evidence.Add(new RuntimeEvidence
{
Signal = "runtime.posture.ttl",
Value = expiresAt.ToString("O", CultureInfo.InvariantCulture)
});
return new RuntimePostureEvaluationResult(posture, evidence);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
logger.LogWarning(ex, "Runtime posture evaluation failed for image {ImageRef}.", imageKey);
if (cached is not null)
{
var cacheSignal = cached.IsExpired(now)
? cached.IsStale(now, staleThreshold) ? "stale-warning" : "stale"
: "hit";
evidence.Add(new RuntimeEvidence
{
Signal = "runtime.posture.cache",
Value = cacheSignal
});
evidence.Add(new RuntimeEvidence
{
Signal = "runtime.posture.error",
Value = ex.GetType().Name
});
return new RuntimePostureEvaluationResult(cached.Posture, evidence);
}
evidence.Add(new RuntimeEvidence
{
Signal = "runtime.posture.error",
Value = ex.GetType().Name
});
return new RuntimePostureEvaluationResult(null, evidence);
}
}
private static string? ResolveImageKey(CriContainerInfo container)
{
if (!string.IsNullOrWhiteSpace(container.ImageRef))
{
return container.ImageRef;
}
return string.IsNullOrWhiteSpace(container.Image) ? null : container.Image;
}
private static RuntimePolicyRequest BuildRequest(CriContainerInfo container, string imageKey)
{
var labels = container.Labels.Count == 0
? null
: new Dictionary<string, string>(container.Labels, StringComparer.Ordinal);
labels?.Remove(CriLabelKeys.PodUid);
return new RuntimePolicyRequest
{
Namespace = container.Labels.TryGetValue(CriLabelKeys.PodNamespace, out var ns) ? ns : null,
Labels = labels,
Images = new[] { imageKey }
};
}
private static RuntimePosture MapPosture(RuntimePolicyImageResult result)
{
var posture = new RuntimePosture
{
ImageSigned = result.Signed,
SbomReferrer = result.HasSbomReferrers ? "present" : "missing"
};
if (result.Rekor is not null)
{
posture = posture with
{
Attestation = new RuntimeAttestation
{
Uuid = result.Rekor.Uuid,
Verified = result.Rekor.Verified
}
};
}
return posture;
}
}

View File

@@ -4,7 +4,6 @@ using StellaOps.Zastava.Observer.Worker;
var builder = Host.CreateApplicationBuilder(args);
builder.Services.AddZastavaRuntimeCore(builder.Configuration, componentName: "observer");
builder.Services.AddHostedService<ObserverBootstrapService>();
builder.Services.AddZastavaObserver(builder.Configuration);
await builder.Build().RunAsync();

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Zastava.Observer.Tests")]

View File

@@ -0,0 +1,287 @@
using System.Buffers.Binary;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Zastava.Observer.Runtime;
internal static class ElfBuildIdReader
{
private const int ElfIdentificationSize = 16;
private const byte ElfClass32 = 1;
private const byte ElfClass64 = 2;
private const byte ElfDataLittleEndian = 1;
private const byte ElfDataBigEndian = 2;
private const uint ProgramHeaderTypeNote = 4;
private const uint NoteTypeGnuBuildId = 3;
private const int Alignment = 4;
private const int MaxNoteSegmentBytes = 1 << 20; // 1 MiB
public static async Task<string?> TryReadBuildIdAsync(string path, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(path))
{
return null;
}
try
{
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
var header = await ReadHeaderAsync(stream, cancellationToken).ConfigureAwait(false);
if (header is null)
{
return null;
}
return await ReadBuildIdFromNotesAsync(stream, header.Value, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException)
{
return null;
}
}
private static async Task<string?> ReadBuildIdFromNotesAsync(Stream stream, ElfHeader header, CancellationToken cancellationToken)
{
if (header.ProgramHeaderEntrySize is 0 || header.ProgramHeaderCount is 0)
{
return null;
}
var entryBuffer = new byte[header.ProgramHeaderEntrySize];
for (var index = 0; index < header.ProgramHeaderCount; index++)
{
var entryOffset = header.ProgramHeaderOffset + (ulong)header.ProgramHeaderEntrySize * (ulong)index;
if (entryOffset > (ulong)stream.Length)
{
break;
}
stream.Seek((long)entryOffset, SeekOrigin.Begin);
if (!await ReadExactlyAsync(stream, entryBuffer.AsMemory(0, header.ProgramHeaderEntrySize), cancellationToken).ConfigureAwait(false))
{
break;
}
var entry = entryBuffer.AsSpan(0, header.ProgramHeaderEntrySize);
var type = ReadUInt32(entry, 0, header.IsLittleEndian);
if (type != ProgramHeaderTypeNote)
{
continue;
}
ulong segmentOffset;
ulong segmentSize;
if (header.Class == ElfClass64)
{
segmentOffset = ReadUInt64(entry, 8, header.IsLittleEndian);
segmentSize = ReadUInt64(entry, 32, header.IsLittleEndian);
}
else
{
segmentOffset = ReadUInt32(entry, 4, header.IsLittleEndian);
segmentSize = ReadUInt32(entry, 16, header.IsLittleEndian);
}
if (segmentSize == 0 || segmentOffset > (ulong)stream.Length)
{
continue;
}
var boundedSize = (int)Math.Min(segmentSize, (ulong)MaxNoteSegmentBytes);
if (boundedSize <= 0)
{
continue;
}
stream.Seek((long)segmentOffset, SeekOrigin.Begin);
var segmentBuffer = new byte[boundedSize];
if (!await ReadExactlyAsync(stream, segmentBuffer.AsMemory(0, boundedSize), cancellationToken).ConfigureAwait(false))
{
continue;
}
var buildId = ParseNoteSegment(segmentBuffer.AsSpan(0, boundedSize), header.IsLittleEndian);
if (buildId is not null)
{
return buildId;
}
}
return null;
}
private static string? ParseNoteSegment(ReadOnlySpan<byte> segment, bool isLittleEndian)
{
var offset = 0;
while (offset + 12 <= segment.Length)
{
var nameSize = ReadUInt32(segment, offset, isLittleEndian);
var descSize = ReadUInt32(segment, offset + 4, isLittleEndian);
var type = ReadUInt32(segment, offset + 8, isLittleEndian);
offset += 12;
if (nameSize > int.MaxValue || descSize > int.MaxValue)
{
return null;
}
var alignedNameSize = Align((int)nameSize);
var alignedDescSize = Align((int)descSize);
if (offset + alignedNameSize + alignedDescSize > segment.Length)
{
return null;
}
var nameBytes = segment.Slice(offset, (int)nameSize);
offset += alignedNameSize;
var descriptorBytes = segment.Slice(offset, (int)descSize);
offset += alignedDescSize;
if (type == NoteTypeGnuBuildId && IsGnuName(nameBytes))
{
return Convert.ToHexString(descriptorBytes).ToLowerInvariant();
}
}
return null;
}
private static bool IsGnuName(ReadOnlySpan<byte> name)
{
var length = name.IndexOf((byte)0);
if (length < 0)
{
length = name.Length;
}
if (length != 3)
{
return false;
}
return name[0] == (byte)'G'
&& name[1] == (byte)'N'
&& name[2] == (byte)'U';
}
private static async Task<ElfHeader?> ReadHeaderAsync(Stream stream, CancellationToken cancellationToken)
{
stream.Seek(0, SeekOrigin.Begin);
var identBuffer = new byte[ElfIdentificationSize];
if (!await ReadExactlyAsync(stream, identBuffer.AsMemory(0, ElfIdentificationSize), cancellationToken).ConfigureAwait(false))
{
return null;
}
var ident = identBuffer.AsSpan();
if (ident[0] != 0x7F || ident[1] != (byte)'E' || ident[2] != (byte)'L' || ident[3] != (byte)'F')
{
return null;
}
var elfClass = ident[4];
if (elfClass != ElfClass32 && elfClass != ElfClass64)
{
return null;
}
var dataEncoding = ident[5];
var isLittleEndian = dataEncoding is ElfDataLittleEndian or 0;
if (dataEncoding == 0)
{
isLittleEndian = true;
}
else if (dataEncoding != ElfDataLittleEndian && dataEncoding != ElfDataBigEndian)
{
return null;
}
var remainingHeaderSize = elfClass == ElfClass64 ? 64 - ElfIdentificationSize : 52 - ElfIdentificationSize;
var buffer = new byte[remainingHeaderSize];
if (!await ReadExactlyAsync(stream, buffer.AsMemory(0, remainingHeaderSize), cancellationToken).ConfigureAwait(false))
{
return null;
}
var span = buffer.AsSpan(0, remainingHeaderSize);
ulong programHeaderOffset;
ushort programHeaderEntrySize;
ushort programHeaderCount;
if (elfClass == ElfClass64)
{
programHeaderOffset = ReadUInt64(span, 16, isLittleEndian);
programHeaderEntrySize = ReadUInt16(span, 38, isLittleEndian);
programHeaderCount = ReadUInt16(span, 40, isLittleEndian);
}
else
{
programHeaderOffset = ReadUInt32(span, 12, isLittleEndian);
programHeaderEntrySize = ReadUInt16(span, 26, isLittleEndian);
programHeaderCount = ReadUInt16(span, 28, isLittleEndian);
}
return new ElfHeader(elfClass, isLittleEndian, programHeaderOffset, programHeaderEntrySize, programHeaderCount);
}
private static uint ReadUInt32(ReadOnlySpan<byte> buffer, int offset, bool isLittleEndian)
{
var slice = buffer.Slice(offset, sizeof(uint));
return isLittleEndian
? BinaryPrimitives.ReadUInt32LittleEndian(slice)
: BinaryPrimitives.ReadUInt32BigEndian(slice);
}
private static ulong ReadUInt64(ReadOnlySpan<byte> buffer, int offset, bool isLittleEndian)
{
var slice = buffer.Slice(offset, sizeof(ulong));
return isLittleEndian
? BinaryPrimitives.ReadUInt64LittleEndian(slice)
: BinaryPrimitives.ReadUInt64BigEndian(slice);
}
private static ushort ReadUInt16(ReadOnlySpan<byte> buffer, int offset, bool isLittleEndian)
{
var slice = buffer.Slice(offset, sizeof(ushort));
return isLittleEndian
? BinaryPrimitives.ReadUInt16LittleEndian(slice)
: BinaryPrimitives.ReadUInt16BigEndian(slice);
}
private static int Align(int value)
=> (value + (Alignment - 1)) & ~(Alignment - 1);
private static async Task<bool> ReadExactlyAsync(Stream stream, Memory<byte> buffer, CancellationToken cancellationToken)
{
var total = 0;
while (total < buffer.Length)
{
var read = await stream.ReadAsync(buffer.Slice(total), cancellationToken).ConfigureAwait(false);
if (read == 0)
{
return false;
}
total += read;
}
return true;
}
private readonly record struct ElfHeader(byte Class, bool IsLittleEndian, ulong ProgramHeaderOffset, ushort ProgramHeaderEntrySize, ushort ProgramHeaderCount)
{
public byte Class { get; } = Class;
public bool IsLittleEndian { get; } = IsLittleEndian;
public ulong ProgramHeaderOffset { get; } = ProgramHeaderOffset;
public ushort ProgramHeaderEntrySize { get; } = ProgramHeaderEntrySize;
public ushort ProgramHeaderCount { get; } = ProgramHeaderCount;
}
}

View File

@@ -0,0 +1,297 @@
using System.Collections.Concurrent;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading.Channels;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Core.Serialization;
using StellaOps.Zastava.Observer.Configuration;
namespace StellaOps.Zastava.Observer.Runtime;
internal interface IRuntimeEventBuffer
{
ValueTask WriteBatchAsync(IReadOnlyList<RuntimeEventEnvelope> envelopes, CancellationToken cancellationToken);
IAsyncEnumerable<RuntimeEventBufferItem> ReadAllAsync(CancellationToken cancellationToken);
}
internal sealed record RuntimeEventBufferItem(
RuntimeEventEnvelope Envelope,
Func<ValueTask> CompleteAsync,
Func<CancellationToken, ValueTask> RequeueAsync);
internal sealed class RuntimeEventBuffer : IRuntimeEventBuffer
{
private static readonly string FileExtension = ".json";
private readonly Channel<string> channel;
private readonly ConcurrentDictionary<string, byte> inFlight = new(StringComparer.OrdinalIgnoreCase);
private readonly object capacityLock = new();
private readonly string spoolPath;
private readonly ILogger<RuntimeEventBuffer> logger;
private readonly TimeProvider timeProvider;
private readonly long maxDiskBytes;
private long currentBytes;
private readonly int capacity;
public RuntimeEventBuffer(
IOptions<ZastavaObserverOptions> observerOptions,
TimeProvider timeProvider,
ILogger<RuntimeEventBuffer> logger)
{
ArgumentNullException.ThrowIfNull(observerOptions);
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
var options = observerOptions.Value ?? throw new ArgumentNullException(nameof(observerOptions));
capacity = Math.Clamp(options.MaxInMemoryBuffer, 16, 65536);
spoolPath = EnsureSpoolDirectory(options.EventBufferPath);
maxDiskBytes = Math.Clamp(options.MaxDiskBufferBytes, 1_048_576L, 1_073_741_824L); // 1 MiB 1 GiB
var channelOptions = new BoundedChannelOptions(capacity)
{
AllowSynchronousContinuations = false,
FullMode = BoundedChannelFullMode.Wait,
SingleReader = false,
SingleWriter = false
};
channel = Channel.CreateBounded<string>(channelOptions);
var existingFiles = Directory.EnumerateFiles(spoolPath, $"*{FileExtension}", SearchOption.TopDirectoryOnly)
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
foreach (var path in existingFiles)
{
var size = TryGetLength(path);
if (size > 0)
{
Interlocked.Add(ref currentBytes, size);
}
// enqueue existing events for replay
if (!channel.Writer.TryWrite(path))
{
_ = channel.Writer.WriteAsync(path);
}
}
if (existingFiles.Length > 0)
{
logger.LogInformation("Runtime event buffer restored {Count} pending events ({Bytes} bytes) from disk spool.",
existingFiles.Length,
Interlocked.Read(ref currentBytes));
}
}
public async ValueTask WriteBatchAsync(IReadOnlyList<RuntimeEventEnvelope> envelopes, CancellationToken cancellationToken)
{
if (envelopes is null || envelopes.Count == 0)
{
return;
}
foreach (var envelope in envelopes)
{
cancellationToken.ThrowIfCancellationRequested();
var payload = ZastavaCanonicalJsonSerializer.SerializeToUtf8Bytes(envelope);
var filePath = await PersistAsync(payload, cancellationToken).ConfigureAwait(false);
await channel.Writer.WriteAsync(filePath, cancellationToken).ConfigureAwait(false);
}
if (envelopes.Count > capacity / 2)
{
logger.LogDebug("Buffered {Count} runtime events; channel capacity {Capacity}.", envelopes.Count, capacity);
}
}
public async IAsyncEnumerable<RuntimeEventBufferItem> ReadAllAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
while (await channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
{
while (channel.Reader.TryRead(out var filePath))
{
cancellationToken.ThrowIfCancellationRequested();
if (!File.Exists(filePath))
{
RemoveMetricsForMissingFile(filePath);
continue;
}
RuntimeEventEnvelope? envelope = null;
try
{
var json = await File.ReadAllTextAsync(filePath, cancellationToken).ConfigureAwait(false);
envelope = ZastavaCanonicalJsonSerializer.Deserialize<RuntimeEventEnvelope>(json);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to read runtime event payload from {Path}; dropping.", filePath);
await DeleteFileSilentlyAsync(filePath).ConfigureAwait(false);
continue;
}
var currentPath = filePath;
inFlight[currentPath] = 0;
yield return new RuntimeEventBufferItem(
envelope,
CompleteAsync(currentPath),
RequeueAsync(currentPath));
}
}
}
private Func<ValueTask> CompleteAsync(string filePath)
=> async () =>
{
try
{
await DeleteFileSilentlyAsync(filePath).ConfigureAwait(false);
}
finally
{
inFlight.TryRemove(filePath, out _);
}
};
private Func<CancellationToken, ValueTask> RequeueAsync(string filePath)
=> async cancellationToken =>
{
inFlight.TryRemove(filePath, out _);
if (!File.Exists(filePath))
{
RemoveMetricsForMissingFile(filePath);
return;
}
await channel.Writer.WriteAsync(filePath, cancellationToken).ConfigureAwait(false);
};
private async Task<string> PersistAsync(byte[] payload, CancellationToken cancellationToken)
{
var timestamp = timeProvider.GetUtcNow().UtcTicks;
var fileName = $"{timestamp:D20}-{Guid.NewGuid():N}{FileExtension}";
var filePath = Path.Combine(spoolPath, fileName);
Directory.CreateDirectory(spoolPath);
await File.WriteAllBytesAsync(filePath, payload, cancellationToken).ConfigureAwait(false);
Interlocked.Add(ref currentBytes, payload.Length);
EnforceCapacity();
return filePath;
}
private void EnforceCapacity()
{
if (Volatile.Read(ref currentBytes) <= maxDiskBytes)
{
return;
}
lock (capacityLock)
{
if (currentBytes <= maxDiskBytes)
{
return;
}
var candidates = Directory.EnumerateFiles(spoolPath, $"*{FileExtension}", SearchOption.TopDirectoryOnly)
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
foreach (var file in candidates)
{
if (currentBytes <= maxDiskBytes)
{
break;
}
if (inFlight.ContainsKey(file))
{
continue;
}
var length = TryGetLength(file);
try
{
File.Delete(file);
if (length > 0)
{
Interlocked.Add(ref currentBytes, -length);
}
logger.LogWarning("Dropped runtime event {FileName} to enforce disk buffer capacity (limit {MaxBytes} bytes).",
Path.GetFileName(file),
maxDiskBytes);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to purge runtime event buffer file {FileName}.", Path.GetFileName(file));
}
}
}
}
private Task DeleteFileSilentlyAsync(string filePath)
{
if (!File.Exists(filePath))
{
return Task.CompletedTask;
}
var length = TryGetLength(filePath);
try
{
File.Delete(filePath);
if (length > 0)
{
Interlocked.Add(ref currentBytes, -length);
}
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to delete runtime event buffer file {FileName}.", Path.GetFileName(filePath));
}
return Task.CompletedTask;
}
private void RemoveMetricsForMissingFile(string filePath)
{
var length = TryGetLength(filePath);
if (length > 0)
{
Interlocked.Add(ref currentBytes, -length);
}
}
private static string EnsureSpoolDirectory(string? value)
{
var path = string.IsNullOrWhiteSpace(value)
? Path.Combine(Path.GetTempPath(), "zastava-observer", "runtime-events")
: value!;
Directory.CreateDirectory(path);
return path;
}
private static long TryGetLength(string path)
{
try
{
var info = new FileInfo(path);
return info.Exists ? info.Length : 0;
}
catch
{
return 0;
}
}
}

View File

@@ -0,0 +1,525 @@
using System.Buffers;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Runtime.CompilerServices;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
namespace StellaOps.Zastava.Observer.Runtime;
internal interface IRuntimeProcessCollector
{
Task<RuntimeProcessCapture?> CollectAsync(CriContainerInfo container, CancellationToken cancellationToken);
}
internal sealed class RuntimeProcessCollector : IRuntimeProcessCollector
{
private static readonly Regex ShellRegex = new(@"(^|/)(ba)?sh$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex PythonRegex = new(@"(^|/)(python)(\d+(\.\d+)*)?$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
private static readonly Regex NodeRegex = new(@"(^|/)(node|npm|npx)$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
private const string SyntheticArgvFile = "<argv>";
private const int MaxInterpreterTargetLength = 512;
private readonly ZastavaObserverOptions options;
private readonly ILogger<RuntimeProcessCollector> logger;
public RuntimeProcessCollector(IOptions<ZastavaObserverOptions> options, ILogger<RuntimeProcessCollector> logger)
{
ArgumentNullException.ThrowIfNull(options);
this.options = options.Value;
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RuntimeProcessCapture?> CollectAsync(CriContainerInfo container, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(container);
if (container.Pid is null or <= 0)
{
logger.LogDebug("Container {ContainerId} lacks PID information; skipping process capture.", container.Id);
return null;
}
var pid = container.Pid.Value;
var procRoot = options.ProcRootPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
var pidDirectory = Path.Combine(procRoot, pid.ToString(CultureInfo.InvariantCulture));
try
{
var process = await ReadProcessAsync(pidDirectory, pid, cancellationToken).ConfigureAwait(false);
if (process is null)
{
logger.LogDebug("No cmdline information available for PID {Pid}; skipping process capture.", pid);
return null;
}
var buildId = await ElfBuildIdReader.TryReadBuildIdAsync(Path.Combine(pidDirectory, "exe"), cancellationToken).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(buildId))
{
process = process with { BuildId = buildId };
}
var (libraries, evidence) = await ReadLibrariesAsync(pidDirectory, cancellationToken).ConfigureAwait(false);
evidence.Insert(0, new RuntimeEvidence
{
Signal = "procfs.cmdline",
Value = $"{pid}:{string.Join(' ', process.Entrypoint)}"
});
if (!string.IsNullOrWhiteSpace(buildId))
{
evidence.Add(new RuntimeEvidence
{
Signal = "procfs.buildId",
Value = buildId
});
}
return new RuntimeProcessCapture(process, libraries, evidence);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to capture process information for container {ContainerId} (PID {Pid}).", container.Id, pid);
return null;
}
}
private async Task<RuntimeProcess?> ReadProcessAsync(string pidDirectory, int pid, CancellationToken cancellationToken)
{
var cmdlinePath = Path.Combine(pidDirectory, "cmdline");
if (!File.Exists(cmdlinePath))
{
return null;
}
var content = await File.ReadAllBytesAsync(cmdlinePath, cancellationToken).ConfigureAwait(false);
if (content.Length == 0)
{
return null;
}
var arguments = ParseCmdline(content, options.MaxEntrypointArguments);
if (arguments.Count == 0)
{
return null;
}
var entryTrace = BuildEntryTrace(arguments);
return new RuntimeProcess
{
Pid = pid,
Entrypoint = arguments,
EntryTrace = entryTrace
};
}
private async Task<(IReadOnlyList<RuntimeLoadedLibrary> Libraries, List<RuntimeEvidence> Evidence)> ReadLibrariesAsync(
string pidDirectory,
CancellationToken cancellationToken)
{
var mapsPath = Path.Combine(pidDirectory, "maps");
var libraries = new List<RuntimeLoadedLibrary>();
var evidence = new List<RuntimeEvidence>();
if (!File.Exists(mapsPath))
{
return (libraries, evidence);
}
var seen = new HashSet<string>(StringComparer.Ordinal);
var limit = Math.Max(1, options.MaxTrackedLibraries);
var perFileLimit = Math.Max(1024L, options.MaxLibraryBytes);
var hashBudget = options.MaxLibraryHashBytes <= 0
? long.MaxValue
: Math.Max(perFileLimit, options.MaxLibraryHashBytes);
long hashedBytes = 0;
var budgetSignaled = false;
await foreach (var line in ReadLinesAsync(mapsPath, cancellationToken))
{
if (!TryParseMapsEntry(line, out var path, out var baseAddress))
{
continue;
}
if (!seen.Add(path))
{
continue;
}
if (libraries.Count >= limit)
{
evidence.Add(new RuntimeEvidence
{
Signal = "procfs.maps.truncated",
Value = $"limit={limit}"
});
break;
}
long length;
long? inode;
try
{
var fileInfo = new FileInfo(path);
length = fileInfo.Length;
inode = TryGetInode(fileInfo);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
evidence.Add(new RuntimeEvidence
{
Signal = "procfs.maps.error",
Value = $"{path}:{ex.GetType().Name}"
});
continue;
}
var sizeExceeded = length > perFileLimit;
string? hash = null;
if (!sizeExceeded && length > 0)
{
var remainingBudget = hashBudget - hashedBytes;
if (remainingBudget <= 0 || length > remainingBudget)
{
if (!budgetSignaled && hashBudget != long.MaxValue)
{
evidence.Add(new RuntimeEvidence
{
Signal = "procfs.maps.hashBudget",
Value = $"limit={hashBudget}"
});
budgetSignaled = true;
}
}
else
{
try
{
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
hash = await ComputeSha256Async(stream, cancellationToken).ConfigureAwait(false);
hashedBytes += length;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
evidence.Add(new RuntimeEvidence
{
Signal = "procfs.maps.error",
Value = $"{path}:{ex.GetType().Name}"
});
}
}
}
if (sizeExceeded)
{
evidence.Add(new RuntimeEvidence
{
Signal = "procfs.maps.skipped",
Value = $"{path}:size>{perFileLimit}"
});
}
var library = new RuntimeLoadedLibrary
{
Path = path,
Inode = inode,
Sha256 = hash
};
libraries.Add(library);
var value = baseAddress is null ? path : $"{path}@{baseAddress}";
evidence.Add(new RuntimeEvidence
{
Signal = "procfs.maps",
Value = value
});
}
evidence.Add(new RuntimeEvidence
{
Signal = "procfs.maps.count",
Value = libraries.Count.ToString(CultureInfo.InvariantCulture)
});
return (libraries, evidence);
}
private static async IAsyncEnumerable<string> ReadLinesAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken)
{
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
using var reader = new StreamReader(stream, Encoding.UTF8);
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var line = await reader.ReadLineAsync().ConfigureAwait(false);
if (line is null)
{
yield break;
}
yield return line;
}
}
private static bool TryParseMapsEntry(string line, out string path, out string? baseAddress)
{
path = string.Empty;
baseAddress = null;
if (string.IsNullOrWhiteSpace(line))
{
return false;
}
var span = line.AsSpan().Trim();
var lastSpace = span.LastIndexOf(' ');
if (lastSpace < 0 || lastSpace >= span.Length - 1)
{
return false;
}
var candidate = span[(lastSpace + 1)..].Trim();
if (candidate.IsEmpty || candidate[0] == '[')
{
return false;
}
path = candidate.ToString();
var firstSpace = span.IndexOf(' ');
if (firstSpace > 0)
{
var rangeSpan = span[..firstSpace];
var dashIndex = rangeSpan.IndexOf('-');
if (dashIndex > 0)
{
var startSpan = rangeSpan[..dashIndex];
if (!startSpan.IsEmpty)
{
baseAddress = startSpan.StartsWith("0x", StringComparison.OrdinalIgnoreCase)
? startSpan.ToString()
: $"0x{startSpan.ToString()}";
}
}
}
return true;
}
private static async Task<string> ComputeSha256Async(Stream stream, CancellationToken cancellationToken)
{
using var sha = SHA256.Create();
var buffer = ArrayPool<byte>.Shared.Rent(8192);
try
{
int read;
while ((read = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken).ConfigureAwait(false)) > 0)
{
sha.TransformBlock(buffer, 0, read, null, 0);
}
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
return Convert.ToHexString(sha.Hash!).ToLowerInvariant();
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
}
private static long? TryGetInode(FileInfo fileInfo) => null;
private static List<string> ParseCmdline(byte[] content, int maxArguments)
{
var segments = Encoding.UTF8.GetString(content).Split('\0', StringSplitOptions.RemoveEmptyEntries);
var list = segments.Take(maxArguments).ToList();
return list;
}
private static IReadOnlyList<RuntimeEntryTrace> BuildEntryTrace(IReadOnlyList<string> arguments)
{
var traces = new List<RuntimeEntryTrace>();
if (arguments.Count == 0)
{
return traces;
}
var first = arguments[0];
traces.Add(new RuntimeEntryTrace
{
File = first,
Op = "exec",
Target = first
});
if (arguments.Count >= 3 && ShellRegex.IsMatch(first) && string.Equals(arguments[1], "-c", StringComparison.Ordinal))
{
var script = arguments[2];
var tokens = TokenizeCommand(script);
if (tokens.Count > 0)
{
traces.Add(new RuntimeEntryTrace
{
File = tokens[0],
Op = "shell",
Target = script
});
TryAddInterpreterTrace(traces, tokens);
}
}
else
{
TryAddInterpreterTrace(traces, arguments);
}
return traces;
}
private static void TryAddInterpreterTrace(List<RuntimeEntryTrace> traces, IReadOnlyList<string> tokens)
{
if (tokens.Count == 0)
{
return;
}
var interpreter = tokens[0];
if (PythonRegex.IsMatch(interpreter))
{
var target = ResolveInterpreterTarget(tokens, 1);
if (!string.IsNullOrEmpty(target))
{
traces.Add(new RuntimeEntryTrace
{
File = SyntheticArgvFile,
Op = "python",
Target = TrimTarget(target!)
});
}
}
else if (NodeRegex.IsMatch(interpreter))
{
var target = ResolveInterpreterTarget(tokens, 1);
if (!string.IsNullOrEmpty(target))
{
traces.Add(new RuntimeEntryTrace
{
File = SyntheticArgvFile,
Op = "node",
Target = TrimTarget(target!)
});
}
}
}
private static string? ResolveInterpreterTarget(IReadOnlyList<string> tokens, int startIndex)
{
for (var i = startIndex; i < tokens.Count; i++)
{
var candidate = tokens[i];
if (string.IsNullOrWhiteSpace(candidate))
{
continue;
}
if (candidate.StartsWith("-", StringComparison.Ordinal))
{
if ((string.Equals(candidate, "-m", StringComparison.Ordinal)
|| string.Equals(candidate, "-c", StringComparison.Ordinal)
|| string.Equals(candidate, "-e", StringComparison.Ordinal))
&& i + 1 < tokens.Count)
{
return tokens[i + 1];
}
continue;
}
return candidate;
}
return null;
}
private static string TrimTarget(string value)
{
if (value.Length <= MaxInterpreterTargetLength)
{
return value;
}
return value[..MaxInterpreterTargetLength];
}
private static List<string> TokenizeCommand(string command)
{
var tokens = new List<string>();
if (string.IsNullOrWhiteSpace(command))
{
return tokens;
}
var current = new StringBuilder();
bool inQuotes = false;
char quoteChar = '"';
foreach (var ch in command)
{
if (inQuotes)
{
if (ch == quoteChar)
{
inQuotes = false;
}
else
{
current.Append(ch);
}
}
else
{
if (ch == '"' || ch == '\'')
{
inQuotes = true;
quoteChar = ch;
}
else if (char.IsWhiteSpace(ch))
{
if (current.Length > 0)
{
tokens.Add(current.ToString());
current.Clear();
}
}
else
{
current.Append(ch);
}
}
}
if (current.Length > 0)
{
tokens.Add(current.ToString());
}
return tokens;
}
}
internal sealed record RuntimeProcessCapture(
RuntimeProcess Process,
IReadOnlyList<RuntimeLoadedLibrary> Libraries,
IReadOnlyList<RuntimeEvidence> Evidence);

View File

@@ -2,8 +2,10 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| ZASTAVA-OBS-12-001 | DOING | Zastava Observer Guild | ZASTAVA-CORE-12-201 | Build container lifecycle watcher that tails CRI (containerd/cri-o/docker) events and emits deterministic runtime records with buffering + backoff. | Fixture cluster produces start/stop events with stable ordering, jitter/backoff tested, metrics/logging wired. |
| ZASTAVA-OBS-12-002 | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-001 | Capture entrypoint traces and loaded libraries, hashing binaries and correlating to SBOM baseline per architecture sections 2.1 and 10. | EntryTrace parser covers shell/python/node launchers, loaded library hashes recorded, fixtures assert linkage to SBOM usage view. |
| ZASTAVA-OBS-12-003 | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Implement runtime posture checks (signature/SBOM/attestation presence) with offline caching and warning surfaces. | Observer marks posture status, caches refresh across restarts, integration tests prove offline tolerance. |
| ZASTAVA-OBS-12-004 | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Batch `/runtime/events` submissions with disk-backed buffer, rate limits, and deterministic envelopes. | Buffered submissions survive restart, rate-limits enforced in tests, JSON envelopes match schema in docs/events. |
| ZASTAVA-OBS-17-005 | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Collect GNU build-id for ELF processes and attach it to emitted runtime events to enable symbol lookup + debug-store correlation. | Observer reads build-id via `/proc/<pid>/exe`/notes without pausing workloads, runtime events include `buildId` field, fixtures cover glibc/musl images, docs updated with retrieval notes. |
| ZASTAVA-OBS-12-001 | DONE (2025-10-24) | Zastava Observer Guild | ZASTAVA-CORE-12-201 | Build container lifecycle watcher that tails CRI (containerd/cri-o/docker) events and emits deterministic runtime records with buffering + backoff. | Fixture cluster produces start/stop events with stable ordering, jitter/backoff tested, metrics/logging wired. |
| ZASTAVA-OBS-12-002 | DONE (2025-10-24) | Zastava Observer Guild | ZASTAVA-OBS-12-001 | Capture entrypoint traces and loaded libraries, hashing binaries and correlating to SBOM baseline per architecture sections 2.1 and 10. | EntryTrace parser covers shell/python/node launchers, loaded library hashes recorded, fixtures assert linkage to SBOM usage view. |
| ZASTAVA-OBS-12-003 | DONE (2025-10-24) | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Implement runtime posture checks (signature/SBOM/attestation presence) with offline caching and warning surfaces. | Observer marks posture status, caches refresh across restarts, integration tests prove offline tolerance. |
| ZASTAVA-OBS-12-004 | DONE (2025-10-24) | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Batch `/runtime/events` submissions with disk-backed buffer, rate limits, and deterministic envelopes. | Buffered submissions survive restart, rate-limits enforced in tests, JSON envelopes match schema in docs/events. |
| ZASTAVA-OBS-17-005 | DOING (2025-10-24) | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Collect GNU build-id for ELF processes and attach it to emitted runtime events to enable symbol lookup + debug-store correlation. | Observer reads build-id via `/proc/<pid>/exe`/notes without pausing workloads, runtime events include `buildId` field, fixtures cover glibc/musl images, docs updated with retrieval notes. |
> 2025-10-24: Observer unit tests pending; `dotnet restore` requires offline copies of `Google.Protobuf`, `Grpc.Net.Client`, `Grpc.Tools` in `local-nuget` before execution can be verified.

View File

@@ -0,0 +1,26 @@
using StellaOps.Zastava.Observer.Configuration;
namespace StellaOps.Zastava.Observer.Worker;
internal static class BackoffCalculator
{
public static TimeSpan ComputeDelay(ObserverBackoffOptions options, int attempt, Random random)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(random);
var cappedAttempt = Math.Max(1, attempt);
var baseDelayMs = options.Initial.TotalMilliseconds * Math.Pow(2, cappedAttempt - 1);
baseDelayMs = Math.Min(baseDelayMs, options.Max.TotalMilliseconds);
if (options.JitterRatio <= 0)
{
return TimeSpan.FromMilliseconds(baseDelayMs);
}
var jitterWindow = baseDelayMs * options.JitterRatio;
var jitter = (random.NextDouble() * 2 - 1) * jitterWindow;
var jittered = Math.Clamp(baseDelayMs + jitter, options.Initial.TotalMilliseconds, options.Max.TotalMilliseconds);
return TimeSpan.FromMilliseconds(jittered);
}
}

View File

@@ -0,0 +1,197 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Core.Configuration;
using StellaOps.Zastava.Core.Diagnostics;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
using StellaOps.Zastava.Observer.Runtime;
namespace StellaOps.Zastava.Observer.Worker;
internal sealed class ContainerLifecycleHostedService : BackgroundService
{
private readonly ICriRuntimeClientFactory clientFactory;
private readonly IOptionsMonitor<ZastavaObserverOptions> observerOptions;
private readonly IOptionsMonitor<ZastavaRuntimeOptions> runtimeOptions;
private readonly IZastavaLogScopeBuilder logScopeBuilder;
private readonly IZastavaRuntimeMetrics runtimeMetrics;
private readonly IRuntimeEventBuffer eventBuffer;
private readonly ContainerStateTrackerFactory trackerFactory;
private readonly ContainerRuntimePoller poller;
private readonly IRuntimeProcessCollector processCollector;
private readonly TimeProvider timeProvider;
private readonly ILogger<ContainerLifecycleHostedService> logger;
private readonly Random jitterRandom = new();
public ContainerLifecycleHostedService(
ICriRuntimeClientFactory clientFactory,
IOptionsMonitor<ZastavaObserverOptions> observerOptions,
IOptionsMonitor<ZastavaRuntimeOptions> runtimeOptions,
IZastavaLogScopeBuilder logScopeBuilder,
IZastavaRuntimeMetrics runtimeMetrics,
IRuntimeEventBuffer eventBuffer,
ContainerStateTrackerFactory trackerFactory,
ContainerRuntimePoller poller,
IRuntimeProcessCollector processCollector,
TimeProvider timeProvider,
ILogger<ContainerLifecycleHostedService> logger)
{
this.clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory));
this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions));
this.runtimeOptions = runtimeOptions ?? throw new ArgumentNullException(nameof(runtimeOptions));
this.logScopeBuilder = logScopeBuilder ?? throw new ArgumentNullException(nameof(logScopeBuilder));
this.runtimeMetrics = runtimeMetrics ?? throw new ArgumentNullException(nameof(runtimeMetrics));
this.eventBuffer = eventBuffer ?? throw new ArgumentNullException(nameof(eventBuffer));
this.trackerFactory = trackerFactory ?? throw new ArgumentNullException(nameof(trackerFactory));
this.poller = poller ?? throw new ArgumentNullException(nameof(poller));
this.processCollector = processCollector ?? throw new ArgumentNullException(nameof(processCollector));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override Task ExecuteAsync(CancellationToken stoppingToken)
{
var options = observerOptions.CurrentValue;
var activeEndpoints = options.Runtimes
.Where(static runtime => runtime.Enabled)
.ToArray();
if (activeEndpoints.Length == 0)
{
logger.LogWarning("No container runtime endpoints configured; lifecycle watcher idle.");
return Task.CompletedTask;
}
var tasks = activeEndpoints
.Select(endpoint => MonitorRuntimeAsync(endpoint, stoppingToken))
.ToArray();
return Task.WhenAll(tasks);
}
private async Task MonitorRuntimeAsync(ContainerRuntimeEndpointOptions endpoint, CancellationToken cancellationToken)
{
var runtime = runtimeOptions.CurrentValue;
var tenant = runtime.Tenant;
var nodeName = observerOptions.CurrentValue.NodeName;
var pollInterval = endpoint.PollInterval ?? observerOptions.CurrentValue.PollInterval;
var backoffOptions = observerOptions.CurrentValue.Backoff;
while (!cancellationToken.IsCancellationRequested)
{
await using var client = clientFactory.Create(endpoint);
CriRuntimeIdentity identity;
try
{
identity = await client.GetIdentityAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
await HandleFailureAsync(endpoint, 1, backoffOptions, ex, cancellationToken).ConfigureAwait(false);
continue;
}
var tracker = trackerFactory.Create();
var failureCount = 0;
while (!cancellationToken.IsCancellationRequested)
{
try
{
var envelopes = await poller.PollAsync(
tracker,
client,
endpoint,
identity,
tenant,
nodeName,
timeProvider,
processCollector,
cancellationToken).ConfigureAwait(false);
if (envelopes.Count > 0)
{
await PublishAsync(endpoint, envelopes, cancellationToken).ConfigureAwait(false);
}
failureCount = 0;
await Task.Delay(pollInterval, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
return;
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
failureCount++;
await HandleFailureAsync(endpoint, failureCount, backoffOptions, ex, cancellationToken).ConfigureAwait(false);
break; // recreate client
}
}
}
}
private async Task PublishAsync(ContainerRuntimeEndpointOptions endpoint, IReadOnlyList<RuntimeEventEnvelope> envelopes, CancellationToken cancellationToken)
{
var endpointName = endpoint.ResolveName();
foreach (var envelope in envelopes)
{
var tags = runtimeMetrics.DefaultTags
.Concat(new[]
{
new KeyValuePair<string, object?>("runtime_endpoint", endpointName),
new KeyValuePair<string, object?>("event_kind", envelope.Event.Kind.ToString().ToLowerInvariant())
})
.ToArray();
runtimeMetrics.RuntimeEvents.Add(1, tags);
var scope = logScopeBuilder.BuildScope(
correlationId: envelope.Event.EventId,
node: envelope.Event.Node,
workload: envelope.Event.Workload.ContainerId,
eventId: envelope.Event.EventId,
additional: new Dictionary<string, string>
{
["runtimeEndpoint"] = endpointName,
["kind"] = envelope.Event.Kind.ToString()
});
using (logger.BeginScope(scope))
{
logger.LogInformation("Observed container {ContainerId} ({Kind}) for node {Node}.",
envelope.Event.Workload.ContainerId,
envelope.Event.Kind,
envelope.Event.Node);
}
}
await eventBuffer.WriteBatchAsync(envelopes, cancellationToken).ConfigureAwait(false);
}
private async Task HandleFailureAsync(
ContainerRuntimeEndpointOptions endpoint,
int failureCount,
ObserverBackoffOptions backoffOptions,
Exception exception,
CancellationToken cancellationToken)
{
var delay = BackoffCalculator.ComputeDelay(backoffOptions, failureCount, jitterRandom);
logger.LogWarning(exception, "Runtime watcher for {Endpoint} encountered error (attempt {Attempt}); retrying after {Delay}.",
endpoint.ResolveName(),
failureCount,
delay);
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
}
}

View File

@@ -0,0 +1,124 @@
using Microsoft.Extensions.Logging;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
using StellaOps.Zastava.Observer.Cri;
using StellaOps.Zastava.Observer.Posture;
using StellaOps.Zastava.Observer.Runtime;
namespace StellaOps.Zastava.Observer.Worker;
internal sealed class ContainerRuntimePoller
{
private readonly ILogger<ContainerRuntimePoller> logger;
private readonly IRuntimePostureEvaluator? postureEvaluator;
public ContainerRuntimePoller(ILogger<ContainerRuntimePoller> logger, IRuntimePostureEvaluator? postureEvaluator = null)
{
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.postureEvaluator = postureEvaluator;
}
public async Task<IReadOnlyList<RuntimeEventEnvelope>> PollAsync(
ContainerStateTracker tracker,
ICriRuntimeClient client,
ContainerRuntimeEndpointOptions endpoint,
CriRuntimeIdentity identity,
string tenant,
string nodeName,
TimeProvider timeProvider,
IRuntimeProcessCollector? processCollector,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(tracker);
ArgumentNullException.ThrowIfNull(client);
ArgumentNullException.ThrowIfNull(endpoint);
ArgumentNullException.ThrowIfNull(identity);
ArgumentNullException.ThrowIfNull(timeProvider);
var pollTimestamp = timeProvider.GetUtcNow();
tracker.BeginCycle();
var runningContainers = await client.ListContainersAsync(ContainerState.ContainerRunning, cancellationToken).ConfigureAwait(false);
var generated = new List<RuntimeEventEnvelope>();
if (runningContainers.Count > 0)
{
foreach (var container in runningContainers)
{
var enriched = container;
var status = await client.GetContainerStatusAsync(container.Id, cancellationToken).ConfigureAwait(false);
if (status is not null)
{
enriched = status;
}
var lifecycleEvent = tracker.MarkRunning(enriched, pollTimestamp);
if (lifecycleEvent is null)
{
continue;
}
RuntimeProcessCapture? capture = null;
if (processCollector is not null && lifecycleEvent.Kind == ContainerLifecycleEventKind.Start)
{
capture = await processCollector.CollectAsync(enriched, cancellationToken).ConfigureAwait(false);
}
RuntimePostureEvaluationResult? posture = null;
if (this.postureEvaluator is not null)
{
posture = await this.postureEvaluator.EvaluateAsync(enriched, cancellationToken).ConfigureAwait(false);
}
generated.Add(RuntimeEventFactory.Create(
lifecycleEvent,
endpoint,
identity,
tenant,
nodeName,
capture,
posture?.Posture,
posture?.Evidence));
}
}
var stopEvents = await tracker.CompleteCycleAsync(
id => client.GetContainerStatusAsync(id, cancellationToken),
pollTimestamp,
cancellationToken).ConfigureAwait(false);
foreach (var lifecycleEvent in stopEvents)
{
RuntimePostureEvaluationResult? posture = null;
if (this.postureEvaluator is not null)
{
posture = await this.postureEvaluator.EvaluateAsync(lifecycleEvent.Snapshot, cancellationToken).ConfigureAwait(false);
}
generated.Add(RuntimeEventFactory.Create(
lifecycleEvent,
endpoint,
identity,
tenant,
nodeName,
null,
posture?.Posture,
posture?.Evidence));
}
if (generated.Count == 0)
{
return Array.Empty<RuntimeEventEnvelope>();
}
var ordered = generated
.OrderBy(static envelope => envelope.Event.When)
.ThenBy(static envelope => envelope.Event.Workload.ContainerId, StringComparer.Ordinal)
.ToArray();
logger.LogDebug("Generated {Count} runtime events for endpoint {EndpointName}.", ordered.Length, endpoint.ResolveName());
return ordered;
}
}

View File

@@ -0,0 +1,225 @@
using System.Linq;
using System.Net;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Observer.Backend;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.Runtime;
namespace StellaOps.Zastava.Observer.Worker;
internal sealed class RuntimeEventDispatchService : BackgroundService
{
private readonly IRuntimeEventBuffer buffer;
private readonly IRuntimeEventsClient eventsClient;
private readonly IOptionsMonitor<ZastavaObserverOptions> observerOptions;
private readonly TimeProvider timeProvider;
private readonly ILogger<RuntimeEventDispatchService> logger;
public RuntimeEventDispatchService(
IRuntimeEventBuffer buffer,
IRuntimeEventsClient eventsClient,
IOptionsMonitor<ZastavaObserverOptions> observerOptions,
TimeProvider timeProvider,
ILogger<RuntimeEventDispatchService> logger)
{
this.buffer = buffer ?? throw new ArgumentNullException(nameof(buffer));
this.eventsClient = eventsClient ?? throw new ArgumentNullException(nameof(eventsClient));
this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var batch = new List<RuntimeEventBufferItem>();
var enumerator = buffer.ReadAllAsync(stoppingToken).GetAsyncEnumerator(stoppingToken);
Task<bool>? moveNextTask = null;
Task? flushDelayTask = null;
CancellationTokenSource? flushDelayCts = null;
try
{
while (!stoppingToken.IsCancellationRequested)
{
moveNextTask ??= enumerator.MoveNextAsync().AsTask();
if (batch.Count > 0 && flushDelayTask is null)
{
StartFlushTimer(ref flushDelayTask, ref flushDelayCts, stoppingToken);
}
Task completedTask;
if (flushDelayTask is null)
{
completedTask = await Task.WhenAny(moveNextTask).ConfigureAwait(false);
}
else
{
completedTask = await Task.WhenAny(moveNextTask, flushDelayTask).ConfigureAwait(false);
}
if (completedTask == moveNextTask)
{
if (!await moveNextTask.ConfigureAwait(false))
{
break;
}
var item = enumerator.Current;
batch.Add(item);
moveNextTask = null;
var options = observerOptions.CurrentValue;
var batchSize = Math.Clamp(options.PublishBatchSize, 1, 512);
if (batch.Count >= batchSize)
{
ResetFlushTimer(ref flushDelayTask, ref flushDelayCts);
await FlushAsync(batch, stoppingToken).ConfigureAwait(false);
}
}
else
{
// flush timer triggered
ResetFlushTimer(ref flushDelayTask, ref flushDelayCts);
if (batch.Count > 0)
{
await FlushAsync(batch, stoppingToken).ConfigureAwait(false);
}
}
}
}
finally
{
ResetFlushTimer(ref flushDelayTask, ref flushDelayCts);
if (batch.Count > 0 && !stoppingToken.IsCancellationRequested)
{
await FlushAsync(batch, stoppingToken).ConfigureAwait(false);
}
if (moveNextTask is not null)
{
try { await moveNextTask.ConfigureAwait(false); }
catch { /* ignored */ }
}
await enumerator.DisposeAsync().ConfigureAwait(false);
}
}
private async Task FlushAsync(List<RuntimeEventBufferItem> batch, CancellationToken cancellationToken)
{
if (batch.Count == 0)
{
return;
}
var request = new RuntimeEventsIngestRequest
{
BatchId = $"obs-{timeProvider.GetUtcNow():yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}",
Events = batch.Select(item => item.Envelope).ToArray()
};
try
{
var result = await eventsClient.PublishAsync(request, cancellationToken).ConfigureAwait(false);
if (result.Success)
{
foreach (var item in batch)
{
await item.CompleteAsync().ConfigureAwait(false);
}
logger.LogInformation("Runtime events batch published (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}).",
request.BatchId,
result.Accepted,
result.Duplicates);
}
else if (result.RateLimited)
{
await RequeueBatchAsync(batch, cancellationToken).ConfigureAwait(false);
await DelayAsync(result.RetryAfter, cancellationToken).ConfigureAwait(false);
}
}
catch (RuntimeEventsException ex) when (!cancellationToken.IsCancellationRequested)
{
logger.LogWarning(ex, "Runtime events publish failed (status={StatusCode}); batch will be retried.", (int)ex.StatusCode);
await RequeueBatchAsync(batch, cancellationToken).ConfigureAwait(false);
var backoff = ex.StatusCode == HttpStatusCode.ServiceUnavailable
? TimeSpan.FromSeconds(5)
: TimeSpan.FromSeconds(2);
await DelayAsync(backoff, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
logger.LogWarning(ex, "Runtime events publish encountered an unexpected error; batch will be retried.");
await RequeueBatchAsync(batch, cancellationToken).ConfigureAwait(false);
await DelayAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
}
finally
{
batch.Clear();
}
}
private async Task RequeueBatchAsync(IEnumerable<RuntimeEventBufferItem> batch, CancellationToken cancellationToken)
{
foreach (var item in batch)
{
try
{
await item.RequeueAsync(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to requeue runtime event {EventId}; dropping.", item.Envelope.Event.EventId);
await item.CompleteAsync().ConfigureAwait(false);
}
}
}
private async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
{
if (delay <= TimeSpan.Zero)
{
return;
}
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
}
}
private void StartFlushTimer(ref Task? flushTask, ref CancellationTokenSource? cts, CancellationToken stoppingToken)
{
var options = observerOptions.CurrentValue;
var flushIntervalSeconds = Math.Clamp(options.PublishFlushIntervalSeconds, 0.1, 30);
var flushInterval = TimeSpan.FromSeconds(flushIntervalSeconds);
cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
flushTask = Task.Delay(flushInterval, cts.Token);
}
private void ResetFlushTimer(ref Task? flushTask, ref CancellationTokenSource? cts)
{
if (cts is not null)
{
try { cts.Cancel(); } catch { /* ignore */ }
cts.Dispose();
cts = null;
}
flushTask = null;
}
}

View File

@@ -0,0 +1,148 @@
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Text;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.ContainerRuntime;
using StellaOps.Zastava.Observer.ContainerRuntime.Cri;
using StellaOps.Zastava.Observer.Runtime;
namespace StellaOps.Zastava.Observer.Worker;
internal static class RuntimeEventFactory
{
public static RuntimeEventEnvelope Create(
ContainerLifecycleEvent lifecycleEvent,
ContainerRuntimeEndpointOptions endpoint,
CriRuntimeIdentity identity,
string tenant,
string nodeName,
RuntimeProcessCapture? capture = null,
RuntimePosture? posture = null,
IReadOnlyList<RuntimeEvidence>? additionalEvidence = null)
{
ArgumentNullException.ThrowIfNull(lifecycleEvent);
ArgumentNullException.ThrowIfNull(endpoint);
ArgumentNullException.ThrowIfNull(identity);
ArgumentNullException.ThrowIfNull(tenant);
ArgumentNullException.ThrowIfNull(nodeName);
var snapshot = lifecycleEvent.Snapshot;
var workloadLabels = snapshot.Labels ?? new Dictionary<string, string>(StringComparer.Ordinal);
var annotations = snapshot.Annotations is null
? new Dictionary<string, string>(StringComparer.Ordinal)
: new Dictionary<string, string>(snapshot.Annotations, StringComparer.Ordinal);
var platform = ResolvePlatform(workloadLabels, endpoint);
var runtimeEvent = new RuntimeEvent
{
EventId = ComputeEventId(nodeName, lifecycleEvent),
When = lifecycleEvent.Timestamp,
Kind = lifecycleEvent.Kind == ContainerLifecycleEventKind.Start
? RuntimeEventKind.ContainerStart
: RuntimeEventKind.ContainerStop,
Tenant = tenant,
Node = nodeName,
Runtime = new RuntimeEngine
{
Engine = endpoint.Engine.ToEngineString(),
Version = identity.RuntimeVersion
},
Workload = new RuntimeWorkload
{
Platform = platform,
Namespace = TryGet(workloadLabels, CriLabelKeys.PodNamespace),
Pod = TryGet(workloadLabels, CriLabelKeys.PodName),
Container = TryGet(workloadLabels, CriLabelKeys.ContainerName) ?? snapshot.Name,
ContainerId = $"{endpoint.Engine.ToEngineString()}://{snapshot.Id}",
ImageRef = ResolveImageRef(snapshot),
Owner = null
},
Process = capture?.Process,
LoadedLibraries = capture?.Libraries ?? Array.Empty<RuntimeLoadedLibrary>(),
Posture = posture,
Evidence = MergeEvidence(capture?.Evidence, additionalEvidence),
Annotations = annotations.Count == 0 ? null : new SortedDictionary<string, string>(annotations, StringComparer.Ordinal)
};
return RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent);
}
private static string ResolvePlatform(IReadOnlyDictionary<string, string> labels, ContainerRuntimeEndpointOptions endpoint)
{
if (labels.ContainsKey(CriLabelKeys.PodName))
{
return "kubernetes";
}
return endpoint.Engine.ToEngineString();
}
private static IReadOnlyList<RuntimeEvidence> MergeEvidence(
IReadOnlyList<RuntimeEvidence>? primary,
IReadOnlyList<RuntimeEvidence>? secondary)
{
if ((primary is null || primary.Count == 0) && (secondary is null || secondary.Count == 0))
{
return Array.Empty<RuntimeEvidence>();
}
if (secondary is null || secondary.Count == 0)
{
return primary ?? Array.Empty<RuntimeEvidence>();
}
if (primary is null || primary.Count == 0)
{
return secondary;
}
var merged = new List<RuntimeEvidence>(primary.Count + secondary.Count);
merged.AddRange(primary);
merged.AddRange(secondary);
return merged;
}
private static string? ResolveImageRef(CriContainerInfo snapshot)
{
if (!string.IsNullOrWhiteSpace(snapshot.ImageRef))
{
return snapshot.ImageRef;
}
return snapshot.Image;
}
private static string? TryGet(IReadOnlyDictionary<string, string> dictionary, string key)
{
if (dictionary.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value;
}
return null;
}
private static string ComputeEventId(string nodeName, ContainerLifecycleEvent lifecycleEvent)
{
var builder = new StringBuilder()
.Append(nodeName)
.Append('|')
.Append(lifecycleEvent.Snapshot.Id)
.Append('|')
.Append(lifecycleEvent.Timestamp.ToUniversalTime().Ticks)
.Append('|')
.Append((int)lifecycleEvent.Kind);
var bytes = Encoding.UTF8.GetBytes(builder.ToString());
Span<byte> hash = stackalloc byte[16];
if (!MD5.TryHashData(bytes, hash, out _))
{
using var md5 = MD5.Create();
hash = md5.ComputeHash(bytes).AsSpan(0, 16);
}
var guid = new Guid(hash);
return guid.ToString("N");
}
}

View File

@@ -0,0 +1,131 @@
using System.Linq;
using System.Text.Json;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Webhook.Backend;
using StellaOps.Zastava.Webhook.Admission;
using Xunit;
namespace StellaOps.Zastava.Webhook.Tests.Admission;
public sealed class AdmissionResponseBuilderTests
{
[Fact]
public void Build_AllowsWhenAllDecisionsPass()
{
using var document = JsonDocument.Parse("""
{
"metadata": { "namespace": "payments" },
"spec": {
"containers": [ { "name": "api", "image": "ghcr.io/example/api:1.0" } ]
}
}
""");
var pod = document.RootElement;
var spec = pod.GetProperty("spec");
var context = new AdmissionRequestContext(
ApiVersion: "admission.k8s.io/v1",
Kind: "AdmissionReview",
Uid: "abc",
Namespace: "payments",
Labels: new Dictionary<string, string>(),
Containers: new[] { new AdmissionContainerReference("api", "ghcr.io/example/api:1.0") },
PodObject: pod,
PodSpec: spec);
var evaluation = new RuntimeAdmissionEvaluation
{
Decisions = new[]
{
new RuntimeAdmissionDecision
{
OriginalImage = "ghcr.io/example/api:1.0",
ResolvedDigest = "ghcr.io/example/api@sha256:deadbeef",
Verdict = PolicyVerdict.Pass,
Allowed = true,
Policy = new RuntimePolicyImageResult
{
PolicyVerdict = PolicyVerdict.Pass,
HasSbom = true,
Signed = true
},
Reasons = Array.Empty<string>(),
FromCache = false,
ResolutionFailed = false
}
},
BackendFailed = false,
FailOpenApplied = false,
FailureReason = null,
TtlSeconds = 300
};
var builder = new AdmissionResponseBuilder();
var (envelope, response) = builder.Build(context, evaluation);
Assert.Equal("admission.k8s.io/v1", response.ApiVersion);
Assert.True(response.Response.Allowed);
Assert.Null(response.Response.Status);
Assert.NotNull(response.Response.AuditAnnotations);
Assert.True(envelope.Decision.Images.First().HasSbomReferrers);
Assert.StartsWith("sha256-", envelope.Decision.PodSpecDigest, StringComparison.Ordinal);
}
[Fact]
public void Build_DeniedIncludesStatusAndWarnings()
{
using var document = JsonDocument.Parse("""
{
"metadata": { "namespace": "ops" },
"spec": {
"containers": [ { "name": "app", "image": "ghcr.io/example/app:latest" } ]
}
}
""");
var pod = document.RootElement;
var spec = pod.GetProperty("spec");
var context = new AdmissionRequestContext(
"admission.k8s.io/v1",
"AdmissionReview",
"uid-123",
"ops",
new Dictionary<string, string>(),
new[] { new AdmissionContainerReference("app", "ghcr.io/example/app:latest") },
pod,
spec);
var evaluation = new RuntimeAdmissionEvaluation
{
Decisions = new[]
{
new RuntimeAdmissionDecision
{
OriginalImage = "ghcr.io/example/app:latest",
ResolvedDigest = null,
Verdict = PolicyVerdict.Fail,
Allowed = false,
Policy = null,
Reasons = new[] { "policy.fail" },
FromCache = false,
ResolutionFailed = true
}
},
BackendFailed = true,
FailOpenApplied = false,
FailureReason = "backend.unavailable",
TtlSeconds = 60
};
var builder = new AdmissionResponseBuilder();
var (_, response) = builder.Build(context, evaluation);
Assert.False(response.Response.Allowed);
Assert.NotNull(response.Response.Status);
Assert.Equal(403, response.Response.Status!.Code);
Assert.NotNull(response.Response.AuditAnnotations);
Assert.Contains("zastava.stellaops/admission", response.Response.AuditAnnotations!.Keys);
}
}

View File

@@ -0,0 +1,102 @@
using System.Text.Json;
using StellaOps.Zastava.Webhook.Admission;
using Xunit;
namespace StellaOps.Zastava.Webhook.Tests.Admission;
public sealed class AdmissionReviewParserTests
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
[Fact]
public void Parse_ValidRequestExtractsContainers()
{
var dto = Deserialize("""
{
"apiVersion": "admission.k8s.io/v1",
"kind": "AdmissionReview",
"request": {
"uid": "abc-123",
"object": {
"metadata": {
"namespace": "payments",
"labels": { "app": "demo" }
},
"spec": {
"containers": [
{ "name": "api", "image": "ghcr.io/example/api:1.2.3" }
],
"initContainers": [
{ "name": "init", "image": "ghcr.io/example/init:1.0" }
]
}
}
}
}
""");
var parser = new AdmissionReviewParser();
var context = parser.Parse(dto);
Assert.Equal("admission.k8s.io/v1", context.ApiVersion);
Assert.Equal("AdmissionReview", context.Kind);
Assert.Equal("abc-123", context.Uid);
Assert.Equal("payments", context.Namespace);
Assert.Equal("demo", context.Labels["app"]);
Assert.Equal(2, context.Containers.Count);
Assert.Contains(context.Containers, c => c.Name == "api" && c.Image == "ghcr.io/example/api:1.2.3");
Assert.Contains(context.Containers, c => c.Name == "init" && c.Image == "ghcr.io/example/init:1.0");
}
[Fact]
public void Parse_UsesRequestNamespaceWhenAvailable()
{
var dto = Deserialize("""
{
"apiVersion": "admission.k8s.io/v1",
"kind": "AdmissionReview",
"request": {
"uid": "uid-456",
"namespace": "critical",
"object": {
"metadata": {
"labels": { }
},
"spec": {
"containers": [ { "name": "app", "image": "ghcr.io/example/app:latest" } ]
}
}
}
}
""");
var parser = new AdmissionReviewParser();
var context = parser.Parse(dto);
Assert.Equal("critical", context.Namespace);
}
[Fact]
public void Parse_ThrowsWhenNoContainers()
{
var dto = Deserialize("""
{
"apiVersion": "admission.k8s.io/v1",
"kind": "AdmissionReview",
"request": {
"uid": "uid-789",
"object": {
"metadata": { "namespace": "ops" },
"spec": { }
}
}
}
""");
var parser = new AdmissionReviewParser();
var ex = Assert.Throws<AdmissionReviewParseException>(() => parser.Parse(dto));
Assert.Equal("admission.review.containers", ex.Code);
}
private static AdmissionReviewRequestDto Deserialize(string json)
=> JsonSerializer.Deserialize<AdmissionReviewRequestDto>(json, SerializerOptions)!;
}

View File

@@ -0,0 +1,266 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Configuration;
using StellaOps.Zastava.Core.Diagnostics;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Webhook.Admission;
using StellaOps.Zastava.Webhook.Backend;
using StellaOps.Zastava.Webhook.Configuration;
using Xunit;
namespace StellaOps.Zastava.Webhook.Tests.Admission;
public sealed class RuntimeAdmissionPolicyServiceTests
{
private const string SampleDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
[Fact]
public async Task EvaluateAsync_UsesCacheOnSubsequentCalls()
{
var timeProvider = new TestTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var policyClient = new StubRuntimePolicyClient(new RuntimePolicyResponse
{
TtlSeconds = 600,
Results = new Dictionary<string, RuntimePolicyImageResult>
{
[SampleDigest] = new RuntimePolicyImageResult
{
PolicyVerdict = PolicyVerdict.Pass,
Signed = true,
HasSbom = true,
Reasons = Array.Empty<string>()
}
}
});
var runtimeMetrics = new StubRuntimeMetrics();
var optionsMonitor = new StaticOptionsMonitor<ZastavaWebhookOptions>(new ZastavaWebhookOptions());
var cache = new RuntimePolicyCache(Options.Create(optionsMonitor.CurrentValue), timeProvider, NullLogger<RuntimePolicyCache>.Instance);
var resolver = new ImageDigestResolver();
var service = new RuntimeAdmissionPolicyService(
policyClient,
resolver,
cache,
optionsMonitor,
runtimeMetrics,
timeProvider,
NullLogger<RuntimeAdmissionPolicyService>.Instance);
var request = new RuntimeAdmissionRequest(
Namespace: "payments",
Labels: new Dictionary<string, string>(),
Images: new[] { $"ghcr.io/example/api@{SampleDigest}" });
var first = await service.EvaluateAsync(request, CancellationToken.None);
Assert.Single(first.Decisions);
Assert.False(first.BackendFailed);
Assert.Equal(600, first.TtlSeconds);
Assert.Equal(1, policyClient.CallCount);
var second = await service.EvaluateAsync(request, CancellationToken.None);
Assert.Single(second.Decisions);
Assert.Equal(1, policyClient.CallCount); // no additional backend call
Assert.True(second.Decisions[0].FromCache);
Assert.Equal(300, second.TtlSeconds);
}
[Fact]
public async Task EvaluateAsync_FailOpenWhenBackendUnavailable()
{
var timeProvider = new TestTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var policyClient = new StubRuntimePolicyClient(new RuntimePolicyException("backend", System.Net.HttpStatusCode.BadGateway));
var options = new ZastavaWebhookOptions
{
Admission = new ZastavaWebhookAdmissionOptions
{
FailOpenByDefault = false,
FailOpenNamespaces = new HashSet<string>(StringComparer.Ordinal) { "payments" }
}
};
var optionsMonitor = new StaticOptionsMonitor<ZastavaWebhookOptions>(options);
var cache = new RuntimePolicyCache(Options.Create(optionsMonitor.CurrentValue), timeProvider, NullLogger<RuntimePolicyCache>.Instance);
var service = new RuntimeAdmissionPolicyService(
policyClient,
new ImageDigestResolver(),
cache,
optionsMonitor,
new StubRuntimeMetrics(),
timeProvider,
NullLogger<RuntimeAdmissionPolicyService>.Instance);
var request = new RuntimeAdmissionRequest(
"payments",
new Dictionary<string, string>(),
new[] { $"ghcr.io/example/api@{SampleDigest}" });
var evaluation = await service.EvaluateAsync(request, CancellationToken.None);
Assert.True(evaluation.BackendFailed);
Assert.True(evaluation.FailOpenApplied);
Assert.Equal(300, evaluation.TtlSeconds);
var decision = Assert.Single(evaluation.Decisions);
Assert.True(decision.Allowed);
Assert.Contains("zastava.fail_open.backend_unavailable", decision.Reasons);
}
[Fact]
public async Task EvaluateAsync_FailClosedWhenNamespaceConfigured()
{
var timeProvider = new TestTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var policyClient = new StubRuntimePolicyClient(new RuntimePolicyException("backend", System.Net.HttpStatusCode.BadGateway));
var options = new ZastavaWebhookOptions
{
Admission = new ZastavaWebhookAdmissionOptions
{
FailOpenByDefault = true,
FailClosedNamespaces = new HashSet<string>(StringComparer.Ordinal) { "critical" }
}
};
var optionsMonitor = new StaticOptionsMonitor<ZastavaWebhookOptions>(options);
var cache = new RuntimePolicyCache(Options.Create(optionsMonitor.CurrentValue), timeProvider, NullLogger<RuntimePolicyCache>.Instance);
var service = new RuntimeAdmissionPolicyService(
policyClient,
new ImageDigestResolver(),
cache,
optionsMonitor,
new StubRuntimeMetrics(),
timeProvider,
NullLogger<RuntimeAdmissionPolicyService>.Instance);
var request = new RuntimeAdmissionRequest(
"critical",
new Dictionary<string, string>(),
new[] { $"ghcr.io/example/api@{SampleDigest}" });
var evaluation = await service.EvaluateAsync(request, CancellationToken.None);
Assert.True(evaluation.BackendFailed);
Assert.False(evaluation.FailOpenApplied);
Assert.Equal(300, evaluation.TtlSeconds);
var decision = Assert.Single(evaluation.Decisions);
Assert.False(decision.Allowed);
Assert.Contains("zastava.backend.unavailable", decision.Reasons);
}
[Fact]
public async Task EvaluateAsync_ResolutionFailureProducesDeny()
{
var timeProvider = new TestTimeProvider(new DateTimeOffset(2025, 10, 24, 12, 0, 0, TimeSpan.Zero));
var policyClient = new StubRuntimePolicyClient(new RuntimePolicyResponse { TtlSeconds = 300 });
var optionsMonitor = new StaticOptionsMonitor<ZastavaWebhookOptions>(new ZastavaWebhookOptions());
var cache = new RuntimePolicyCache(Options.Create(optionsMonitor.CurrentValue), timeProvider, NullLogger<RuntimePolicyCache>.Instance);
var service = new RuntimeAdmissionPolicyService(
policyClient,
new ImageDigestResolver(),
cache,
optionsMonitor,
new StubRuntimeMetrics(),
timeProvider,
NullLogger<RuntimeAdmissionPolicyService>.Instance);
var request = new RuntimeAdmissionRequest(
Namespace: "payments",
Labels: new Dictionary<string, string>(),
Images: new[] { "ghcr.io/example/api:latest" });
var evaluation = await service.EvaluateAsync(request, CancellationToken.None);
Assert.Equal(300, evaluation.TtlSeconds);
var decision = Assert.Single(evaluation.Decisions);
Assert.False(decision.Allowed);
Assert.True(decision.ResolutionFailed);
Assert.Contains("image.reference.tag_unresolved", decision.Reasons);
}
private sealed class StubRuntimePolicyClient : IRuntimePolicyClient
{
private readonly RuntimePolicyResponse? response;
private readonly Exception? exception;
public StubRuntimePolicyClient(RuntimePolicyResponse response)
{
this.response = response;
}
public StubRuntimePolicyClient(Exception exception)
{
this.exception = exception;
}
public int CallCount { get; private set; }
public Task<RuntimePolicyResponse> EvaluateAsync(RuntimePolicyRequest request, CancellationToken cancellationToken)
{
CallCount++;
if (exception is not null)
{
throw exception;
}
return Task.FromResult(response ?? new RuntimePolicyResponse());
}
}
private sealed class StubRuntimeMetrics : IZastavaRuntimeMetrics
{
public StubRuntimeMetrics()
{
Meter = new Meter("Test.Zastava.Webhook");
RuntimeEvents = Meter.CreateCounter<long>("test.runtime.events");
AdmissionDecisions = Meter.CreateCounter<long>("test.admission.decisions");
BackendLatencyMs = Meter.CreateHistogram<double>("test.backend.latency");
DefaultTags = Array.Empty<KeyValuePair<string, object?>>();
}
public Meter Meter { get; }
public Counter<long> RuntimeEvents { get; }
public Counter<long> AdmissionDecisions { get; }
public Histogram<double> BackendLatencyMs { get; }
public IReadOnlyList<KeyValuePair<string, object?>> DefaultTags { get; }
public void Dispose() => Meter.Dispose();
}
private sealed class StaticOptionsMonitor<T> : IOptionsMonitor<T>
{
public StaticOptionsMonitor(T currentValue)
{
CurrentValue = currentValue;
}
public T CurrentValue { get; }
public T Get(string? name) => CurrentValue;
public IDisposable OnChange(Action<T, string?> listener) => NullDisposable.Instance;
private sealed class NullDisposable : IDisposable
{
public static readonly NullDisposable Instance = new();
public void Dispose()
{
}
}
}
private sealed class TestTimeProvider : TimeProvider
{
private DateTimeOffset now;
public TestTimeProvider(DateTimeOffset initial)
{
now = initial;
}
public override DateTimeOffset GetUtcNow() => now;
public void Advance(TimeSpan delta) => now = now.Add(delta);
}
}

View File

@@ -0,0 +1,97 @@
using System.Globalization;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using StellaOps.Zastava.Core.Diagnostics;
namespace StellaOps.Zastava.Webhook.Admission;
internal static class AdmissionEndpoint
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
public static async Task<IResult> HandleAsync(
HttpContext httpContext,
AdmissionReviewParser parser,
AdmissionResponseBuilder responseBuilder,
IRuntimeAdmissionPolicyService policyService,
IZastavaLogScopeBuilder logScopeBuilder,
ILogger<AdmissionEndpointMarker> logger,
CancellationToken cancellationToken)
{
AdmissionReviewRequestDto? dto;
try
{
dto = await httpContext.Request.ReadFromJsonAsync<AdmissionReviewRequestDto>(SerializerOptions, cancellationToken).ConfigureAwait(false);
}
catch (JsonException ex)
{
logger.LogWarning(ex, "Failed to deserialize AdmissionReview payload.");
return Results.Problem(
statusCode: StatusCodes.Status400BadRequest,
title: "Invalid AdmissionReview",
detail: "Request body was not a valid AdmissionReview document.",
type: "https://stellaops.org/problems/admission.review.invalid-json");
}
AdmissionRequestContext context;
try
{
context = parser.Parse(dto!);
}
catch (AdmissionReviewParseException ex)
{
logger.LogWarning("AdmissionReview parse failure ({Code}): {Message}", ex.Code, ex.Message);
return Results.Problem(
statusCode: StatusCodes.Status400BadRequest,
title: "Invalid AdmissionReview",
detail: ex.Message,
type: $"https://stellaops.org/problems/{ex.Code}");
}
using var scope = logger.BeginScope(logScopeBuilder.BuildScope(
correlationId: context.Uid,
node: null,
workload: context.Namespace,
eventId: context.Uid,
additional: new Dictionary<string, string>
{
["namespace"] = context.Namespace,
["containerCount"] = context.Containers.Count.ToString(CultureInfo.InvariantCulture)
}));
var request = new RuntimeAdmissionRequest(
context.Namespace,
context.Labels,
context.Containers.Select(static c => c.Image).ToArray());
RuntimeAdmissionEvaluation evaluation;
try
{
evaluation = await policyService.EvaluateAsync(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
logger.LogError(ex, "Admission evaluation failed unexpectedly.");
return Results.Problem(
statusCode: StatusCodes.Status500InternalServerError,
title: "Admission evaluation failed",
detail: "An unexpected error occurred while evaluating admission policy.",
type: "https://stellaops.org/problems/admission.evaluation.failed");
}
var (envelope, response) = responseBuilder.Build(context, evaluation);
var allowed = evaluation.Decisions.All(static d => d.Allowed);
logger.LogInformation("Admission decision computed (allowed={Allowed}, containers={Count}, failOpen={FailOpen}).",
allowed,
context.Containers.Count,
evaluation.FailOpenApplied);
httpContext.Response.ContentType = "application/json";
return Results.Json(response, SerializerOptions);
}
}
internal sealed class AdmissionEndpointMarker
{
}

View File

@@ -0,0 +1,15 @@
using System.Text.Json;
namespace StellaOps.Zastava.Webhook.Admission;
internal sealed record AdmissionRequestContext(
string ApiVersion,
string Kind,
string Uid,
string Namespace,
IReadOnlyDictionary<string, string> Labels,
IReadOnlyList<AdmissionContainerReference> Containers,
JsonElement PodObject,
JsonElement PodSpec);
internal sealed record AdmissionContainerReference(string Name, string Image);

View File

@@ -0,0 +1,219 @@
using System.Buffers;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Core.Hashing;
using StellaOps.Zastava.Core.Serialization;
namespace StellaOps.Zastava.Webhook.Admission;
internal sealed class AdmissionResponseBuilder
{
public (AdmissionDecisionEnvelope Envelope, AdmissionReviewResponseDto Response) Build(
AdmissionRequestContext context,
RuntimeAdmissionEvaluation evaluation)
{
var decision = BuildDecision(context, evaluation);
var envelope = AdmissionDecisionEnvelope.Create(decision, ZastavaContractVersions.AdmissionDecision);
var auditAnnotations = CreateAuditAnnotations(envelope, evaluation);
var warnings = BuildWarnings(evaluation);
var allowed = evaluation.Decisions.All(static d => d.Allowed);
var status = allowed
? null
: new AdmissionReviewStatus
{
Code = 403,
Message = BuildFailureMessage(evaluation)
};
var response = new AdmissionReviewResponseDto
{
ApiVersion = context.ApiVersion,
Kind = context.Kind,
Response = new AdmissionReviewResponsePayload
{
Uid = context.Uid,
Allowed = allowed,
Status = status,
Warnings = warnings,
AuditAnnotations = auditAnnotations
}
};
return (envelope, response);
}
private static AdmissionDecision BuildDecision(AdmissionRequestContext context, RuntimeAdmissionEvaluation evaluation)
{
var images = new List<AdmissionImageVerdict>(evaluation.Decisions.Count);
for (var i = 0; i < evaluation.Decisions.Count; i++)
{
var decision = evaluation.Decisions[i];
var container = context.Containers[Math.Min(i, context.Containers.Count - 1)];
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["image"] = decision.OriginalImage
};
if (!string.Equals(container.Image, container.Name, StringComparison.Ordinal))
{
metadata["container"] = container.Name;
}
if (decision.FromCache)
{
metadata["cache"] = "hit";
}
var resolved = decision.ResolvedDigest ?? decision.OriginalImage;
images.Add(new AdmissionImageVerdict
{
Name = container.Name,
Resolved = resolved,
Signed = decision.Policy?.Signed ?? false,
HasSbomReferrers = decision.Policy?.HasSbom ?? false,
PolicyVerdict = decision.Verdict,
Reasons = decision.Reasons,
Rekor = decision.Policy?.Rekor,
Metadata = metadata
});
}
return new AdmissionDecision
{
AdmissionId = context.Uid,
Namespace = context.Namespace,
PodSpecDigest = ComputePodSpecDigest(context.PodSpec),
Images = images,
Decision = evaluation.Decisions.All(static d => d.Allowed)
? AdmissionDecisionOutcome.Allow
: AdmissionDecisionOutcome.Deny,
TtlSeconds = Math.Max(0, evaluation.TtlSeconds),
Annotations = BuildAnnotations(evaluation)
};
}
private static IReadOnlyDictionary<string, string>? BuildAnnotations(RuntimeAdmissionEvaluation evaluation)
{
if (!evaluation.BackendFailed && !evaluation.FailOpenApplied && evaluation.FailureReason is null)
{
return null;
}
var annotations = new Dictionary<string, string>(StringComparer.Ordinal);
if (evaluation.BackendFailed)
{
annotations["zastava.backend.failed"] = "true";
}
if (evaluation.FailOpenApplied)
{
annotations["zastava.failOpen"] = "true";
}
if (!string.IsNullOrWhiteSpace(evaluation.FailureReason))
{
annotations["zastava.failureReason"] = evaluation.FailureReason!;
}
return annotations;
}
private static IReadOnlyDictionary<string, string> CreateAuditAnnotations(AdmissionDecisionEnvelope envelope, RuntimeAdmissionEvaluation evaluation)
{
var annotations = new Dictionary<string, string>(StringComparer.Ordinal)
{
["zastava.stellaops/admission"] = ZastavaCanonicalJsonSerializer.Serialize(envelope)
};
if (evaluation.FailOpenApplied)
{
annotations["zastava.stellaops/failOpen"] = "true";
}
return annotations;
}
private static IReadOnlyList<string>? BuildWarnings(RuntimeAdmissionEvaluation evaluation)
{
var warnings = new List<string>();
if (evaluation.FailOpenApplied)
{
warnings.Add("zastava.fail_open.applied");
}
foreach (var decision in evaluation.Decisions)
{
if (decision.Verdict == PolicyVerdict.Warn)
{
warnings.Add($"policy.warn:{decision.OriginalImage}");
}
}
return warnings.Count == 0 ? null : warnings;
}
private static string BuildFailureMessage(RuntimeAdmissionEvaluation evaluation)
{
if (!string.IsNullOrWhiteSpace(evaluation.FailureReason))
{
return evaluation.FailureReason!;
}
var denied = evaluation.Decisions
.Where(static d => !d.Allowed)
.SelectMany(static d => d.Reasons)
.Distinct(StringComparer.Ordinal)
.ToArray();
return denied.Length > 0
? string.Join(", ", denied)
: "admission.denied";
}
private static string ComputePodSpecDigest(JsonElement podSpec)
{
var buffer = new ArrayBufferWriter<byte>();
using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = false
}))
{
WriteCanonical(podSpec, writer);
}
return ZastavaHashing.ComputeMultihash(buffer.WrittenSpan);
}
private static void WriteCanonical(JsonElement element, Utf8JsonWriter writer)
{
switch (element.ValueKind)
{
case JsonValueKind.Object:
writer.WriteStartObject();
foreach (var property in element.EnumerateObject().OrderBy(static p => p.Name, StringComparer.Ordinal))
{
writer.WritePropertyName(property.Name);
WriteCanonical(property.Value, writer);
}
writer.WriteEndObject();
break;
case JsonValueKind.Array:
writer.WriteStartArray();
foreach (var item in element.EnumerateArray())
{
WriteCanonical(item, writer);
}
writer.WriteEndArray();
break;
default:
element.WriteTo(writer);
break;
}
}
}

View File

@@ -0,0 +1,88 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Zastava.Webhook.Admission;
internal sealed record AdmissionReviewRequestDto
{
[JsonPropertyName("apiVersion")]
public string? ApiVersion { get; init; }
[JsonPropertyName("kind")]
public string? Kind { get; init; }
[JsonPropertyName("request")]
public AdmissionReviewRequestPayload? Request { get; init; }
}
internal sealed record AdmissionReviewRequestPayload
{
[JsonPropertyName("uid")]
public string? Uid { get; init; }
[JsonPropertyName("kind")]
public AdmissionReviewGroupVersionKind? Kind { get; init; }
[JsonPropertyName("namespace")]
public string? Namespace { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("object")]
public JsonElement Object { get; init; }
[JsonPropertyName("dryRun")]
public bool? DryRun { get; init; }
}
internal sealed record AdmissionReviewGroupVersionKind
{
[JsonPropertyName("group")]
public string? Group { get; init; }
[JsonPropertyName("version")]
public string? Version { get; init; }
[JsonPropertyName("kind")]
public string? Kind { get; init; }
}
internal sealed record AdmissionReviewResponseDto
{
[JsonPropertyName("apiVersion")]
public string ApiVersion { get; init; } = "admission.k8s.io/v1";
[JsonPropertyName("kind")]
public string Kind { get; init; } = "AdmissionReview";
[JsonPropertyName("response")]
public required AdmissionReviewResponsePayload Response { get; init; }
}
internal sealed record AdmissionReviewResponsePayload
{
[JsonPropertyName("uid")]
public required string Uid { get; init; }
[JsonPropertyName("allowed")]
public required bool Allowed { get; init; }
[JsonPropertyName("status")]
public AdmissionReviewStatus? Status { get; init; }
[JsonPropertyName("warnings")]
public IReadOnlyList<string>? Warnings { get; init; }
[JsonPropertyName("auditAnnotations")]
public IReadOnlyDictionary<string, string>? AuditAnnotations { get; init; }
}
internal sealed record AdmissionReviewStatus
{
[JsonPropertyName("code")]
public int? Code { get; init; }
[JsonPropertyName("message")]
public string? Message { get; init; }
}

View File

@@ -0,0 +1,154 @@
using System.Text.Json;
namespace StellaOps.Zastava.Webhook.Admission;
internal sealed class AdmissionReviewParser
{
public AdmissionRequestContext Parse(AdmissionReviewRequestDto dto)
{
if (dto is null)
{
throw new AdmissionReviewParseException("admission.review.invalid", "AdmissionReview payload was empty.");
}
if (!string.Equals(dto.Kind, "AdmissionReview", StringComparison.OrdinalIgnoreCase))
{
throw new AdmissionReviewParseException("admission.review.kind", "AdmissionReview.kind must equal 'AdmissionReview'.");
}
if (string.IsNullOrWhiteSpace(dto.ApiVersion))
{
throw new AdmissionReviewParseException("admission.review.apiVersion", "AdmissionReview.apiVersion is required.");
}
var payload = dto.Request ?? throw new AdmissionReviewParseException("admission.review.request", "AdmissionReview.request is required.");
if (string.IsNullOrWhiteSpace(payload.Uid))
{
throw new AdmissionReviewParseException("admission.review.uid", "AdmissionReview.request.uid is required.");
}
if (payload.Object.ValueKind is not JsonValueKind.Object)
{
throw new AdmissionReviewParseException("admission.review.object", "AdmissionReview.request.object must be a JSON object.");
}
var podObject = payload.Object;
if (!podObject.TryGetProperty("spec", out var podSpec) || podSpec.ValueKind is not JsonValueKind.Object)
{
throw new AdmissionReviewParseException("admission.review.podSpec", "AdmissionReview.request.object.spec is required.");
}
var podNamespace = payload.Namespace
?? TryGetProperty(podObject, "metadata", "namespace")
?? throw new AdmissionReviewParseException("admission.review.namespace", "Namespace could not be determined for the pod.");
var labels = ReadLabels(podObject);
var containers = ReadContainers(podSpec);
if (containers.Count == 0)
{
throw new AdmissionReviewParseException("admission.review.containers", "No containers were found in the pod spec.");
}
return new AdmissionRequestContext(
ApiVersion: dto.ApiVersion!,
Kind: dto.Kind!,
Uid: payload.Uid!,
Namespace: podNamespace,
Labels: labels,
Containers: containers,
PodObject: podObject,
PodSpec: podSpec);
}
private static IReadOnlyDictionary<string, string> ReadLabels(JsonElement podObject)
{
if (!podObject.TryGetProperty("metadata", out var metadata) || metadata.ValueKind is not JsonValueKind.Object)
{
return new Dictionary<string, string>(StringComparer.Ordinal);
}
if (!metadata.TryGetProperty("labels", out var labelsElement) || labelsElement.ValueKind is not JsonValueKind.Object)
{
return new Dictionary<string, string>(StringComparer.Ordinal);
}
var labels = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var property in labelsElement.EnumerateObject())
{
if (property.Value.ValueKind is JsonValueKind.String)
{
labels[property.Name] = property.Value.GetString() ?? string.Empty;
}
}
return labels;
}
private static IReadOnlyList<AdmissionContainerReference> ReadContainers(JsonElement podSpec)
{
var containers = new List<AdmissionContainerReference>();
CollectContainers(podSpec, "containers", containers);
CollectContainers(podSpec, "initContainers", containers);
CollectContainers(podSpec, "ephemeralContainers", containers);
return containers;
}
private static void CollectContainers(JsonElement spec, string propertyName, ICollection<AdmissionContainerReference> sink)
{
if (!spec.TryGetProperty(propertyName, out var array) || array.ValueKind is not JsonValueKind.Array)
{
return;
}
foreach (var element in array.EnumerateArray())
{
if (element.ValueKind is not JsonValueKind.Object)
{
continue;
}
var image = TryGetProperty(element, "image");
if (string.IsNullOrWhiteSpace(image))
{
continue;
}
var name = TryGetProperty(element, "name") ?? image;
sink.Add(new AdmissionContainerReference(name, image));
}
}
private static string? TryGetProperty(JsonElement element, string propertyName)
{
if (element.ValueKind is not JsonValueKind.Object)
{
return null;
}
return element.TryGetProperty(propertyName, out var property) && property.ValueKind is JsonValueKind.String
? property.GetString()
: null;
}
private static string? TryGetProperty(JsonElement element, string firstProperty, string nestedProperty)
{
if (!element.TryGetProperty(firstProperty, out var nested) || nested.ValueKind is not JsonValueKind.Object)
{
return null;
}
return TryGetProperty(nested, nestedProperty);
}
}
internal sealed class AdmissionReviewParseException : Exception
{
public AdmissionReviewParseException(string code, string message)
: base(message)
{
Code = code;
}
public string Code { get; }
}

View File

@@ -0,0 +1,52 @@
using System.Text.RegularExpressions;
namespace StellaOps.Zastava.Webhook.Admission;
internal interface IImageDigestResolver
{
Task<ImageResolutionResult> ResolveAsync(string imageReference, CancellationToken cancellationToken);
}
internal sealed class ImageDigestResolver : IImageDigestResolver
{
private static readonly Regex DigestPattern = new(@"(?<algorithm>[a-z0-9_+.-]+):(?<digest>[a-f0-9]{32,})", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
public Task<ImageResolutionResult> ResolveAsync(string imageReference, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(imageReference))
{
return Task.FromResult(ImageResolutionResult.CreateFailure(imageReference, "image.reference.empty"));
}
if (imageReference.Contains('@', StringComparison.Ordinal))
{
var digest = imageReference[(imageReference.IndexOf('@') + 1)..];
if (DigestPattern.IsMatch(digest))
{
return Task.FromResult(ImageResolutionResult.CreateSuccess(imageReference, digest));
}
return Task.FromResult(ImageResolutionResult.CreateFailure(imageReference, "image.reference.invalid_digest"));
}
if (DigestPattern.IsMatch(imageReference))
{
return Task.FromResult(ImageResolutionResult.CreateSuccess(imageReference, imageReference));
}
return Task.FromResult(ImageResolutionResult.CreateFailure(imageReference, "image.reference.tag_unresolved"));
}
}
internal sealed record ImageResolutionResult(
string Original,
string? ResolvedDigest,
bool Success,
string? FailureReason)
{
public static ImageResolutionResult CreateSuccess(string original, string digest)
=> new(original, digest, true, null);
public static ImageResolutionResult CreateFailure(string original, string reason)
=> new(original, null, false, reason);
}

View File

@@ -0,0 +1,312 @@
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Core.Diagnostics;
using StellaOps.Zastava.Webhook.Backend;
using StellaOps.Zastava.Webhook.Configuration;
namespace StellaOps.Zastava.Webhook.Admission;
internal interface IRuntimeAdmissionPolicyService
{
Task<RuntimeAdmissionEvaluation> EvaluateAsync(RuntimeAdmissionRequest request, CancellationToken cancellationToken);
}
internal sealed class RuntimeAdmissionPolicyService : IRuntimeAdmissionPolicyService
{
private readonly IRuntimePolicyClient policyClient;
private readonly IImageDigestResolver digestResolver;
private readonly RuntimePolicyCache cache;
private readonly IOptionsMonitor<ZastavaWebhookOptions> options;
private readonly IZastavaRuntimeMetrics runtimeMetrics;
private readonly TimeProvider timeProvider;
private readonly ILogger<RuntimeAdmissionPolicyService> logger;
public RuntimeAdmissionPolicyService(
IRuntimePolicyClient policyClient,
IImageDigestResolver digestResolver,
RuntimePolicyCache cache,
IOptionsMonitor<ZastavaWebhookOptions> options,
IZastavaRuntimeMetrics runtimeMetrics,
TimeProvider timeProvider,
ILogger<RuntimeAdmissionPolicyService> logger)
{
this.policyClient = policyClient ?? throw new ArgumentNullException(nameof(policyClient));
this.digestResolver = digestResolver ?? throw new ArgumentNullException(nameof(digestResolver));
this.cache = cache ?? throw new ArgumentNullException(nameof(cache));
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.runtimeMetrics = runtimeMetrics ?? throw new ArgumentNullException(nameof(runtimeMetrics));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<RuntimeAdmissionEvaluation> EvaluateAsync(RuntimeAdmissionRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
if (request.Images.Count == 0)
{
return RuntimeAdmissionEvaluation.Empty();
}
var admissionOptions = options.CurrentValue.Admission;
var resolutionResults = new List<ImageResolutionResult>(request.Images.Count);
foreach (var image in request.Images)
{
var resolution = await digestResolver.ResolveAsync(image, cancellationToken).ConfigureAwait(false);
resolutionResults.Add(resolution);
}
var resolved = resolutionResults.Where(static r => r.Success && r.ResolvedDigest is not null)
.GroupBy(r => r.ResolvedDigest!, StringComparer.Ordinal)
.Select(group => new ResolvedDigest(group.Key, group.ToArray()))
.ToArray();
var combinedResults = new Dictionary<string, RuntimePolicyImageResult>(StringComparer.Ordinal);
var backendMisses = new List<string>();
var fromCache = new HashSet<string>(StringComparer.Ordinal);
foreach (var digest in resolved)
{
if (cache.TryGet(digest.Digest, out var cached))
{
combinedResults[digest.Digest] = cached;
fromCache.Add(digest.Digest);
}
else
{
backendMisses.Add(digest.Digest);
}
}
RuntimePolicyResponse? backendResponse = null;
bool backendFailed = false;
var ttlSeconds = 300;
if (backendMisses.Count > 0)
{
try
{
backendResponse = await policyClient.EvaluateAsync(new RuntimePolicyRequest
{
Namespace = request.Namespace ?? string.Empty,
Labels = request.Labels,
Images = backendMisses
}, cancellationToken).ConfigureAwait(false);
var now = timeProvider.GetUtcNow();
var expiry = CalculateExpiry(backendResponse);
ttlSeconds = Math.Max(1, (int)Math.Ceiling((expiry - now).TotalSeconds));
foreach (var pair in backendResponse.Results)
{
combinedResults[pair.Key] = pair.Value;
cache.Set(pair.Key, pair.Value, expiry);
}
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
backendFailed = true;
logger.LogWarning(ex, "Runtime policy backend call failed for namespace {Namespace}.", request.Namespace ?? "<none>");
}
}
var failOpenApplied = false;
var decisions = new List<RuntimeAdmissionDecision>(request.Images.Count);
var effectiveTtl = backendResponse?.TtlSeconds is > 0 ? backendResponse.TtlSeconds : ttlSeconds;
if (backendFailed && backendMisses.Count > 0)
{
failOpenApplied = ShouldFailOpen(admissionOptions, request.Namespace);
foreach (var resolution in resolutionResults)
{
if (resolution.Success && resolution.ResolvedDigest is not null)
{
var allowed = failOpenApplied;
var reasons = failOpenApplied
? new[] { "zastava.fail_open.backend_unavailable" }
: new[] { "zastava.backend.unavailable" };
RecordDecisionMetrics(allowed, true, failOpenApplied, RuntimeEventKind.ContainerStart);
decisions.Add(new RuntimeAdmissionDecision
{
OriginalImage = resolution.Original,
ResolvedDigest = resolution.ResolvedDigest,
Verdict = allowed ? PolicyVerdict.Warn : PolicyVerdict.Error,
Allowed = allowed,
Policy = null,
Reasons = reasons,
FromCache = false,
ResolutionFailed = false
});
}
else
{
decisions.Add(CreateResolutionFailureDecision(resolution));
}
}
return new RuntimeAdmissionEvaluation
{
Decisions = decisions,
BackendFailed = true,
FailOpenApplied = failOpenApplied,
FailureReason = failOpenApplied ? null : "backend.unavailable",
TtlSeconds = effectiveTtl
};
}
foreach (var resolution in resolutionResults)
{
if (!resolution.Success || resolution.ResolvedDigest is null)
{
var failureDecision = CreateResolutionFailureDecision(resolution);
RecordDecisionMetrics(failureDecision.Allowed, false, false, RuntimeEventKind.ContainerStart);
decisions.Add(failureDecision);
continue;
}
if (!combinedResults.TryGetValue(resolution.ResolvedDigest, out var policyResult))
{
var synthetic = new RuntimeAdmissionDecision
{
OriginalImage = resolution.Original,
ResolvedDigest = resolution.ResolvedDigest,
Verdict = PolicyVerdict.Error,
Allowed = false,
Policy = null,
Reasons = new[] { "zastava.policy.result.missing" },
FromCache = false,
ResolutionFailed = false
};
RecordDecisionMetrics(false, false, false, RuntimeEventKind.ContainerStart);
decisions.Add(synthetic);
continue;
}
var allowed = policyResult.PolicyVerdict is PolicyVerdict.Pass or PolicyVerdict.Warn;
var cached = fromCache.Contains(resolution.ResolvedDigest);
var reasons = policyResult.Reasons.Count > 0 ? policyResult.Reasons : Array.Empty<string>();
RecordDecisionMetrics(allowed, cached, false, RuntimeEventKind.ContainerStart);
decisions.Add(new RuntimeAdmissionDecision
{
OriginalImage = resolution.Original,
ResolvedDigest = resolution.ResolvedDigest,
Verdict = policyResult.PolicyVerdict,
Allowed = allowed,
Policy = policyResult,
Reasons = reasons,
FromCache = cached,
ResolutionFailed = false
});
}
return new RuntimeAdmissionEvaluation
{
Decisions = decisions,
BackendFailed = backendFailed,
FailOpenApplied = failOpenApplied,
FailureReason = null,
TtlSeconds = effectiveTtl
};
}
private static RuntimeAdmissionDecision CreateResolutionFailureDecision(ImageResolutionResult resolution)
=> new RuntimeAdmissionDecision
{
OriginalImage = resolution.Original,
ResolvedDigest = null,
Verdict = PolicyVerdict.Fail,
Allowed = false,
Policy = null,
Reasons = new[] { resolution.FailureReason ?? "image.resolution.failed" },
FromCache = false,
ResolutionFailed = true
};
private void RecordDecisionMetrics(bool allowed, bool fromCache, bool failOpen, RuntimeEventKind eventKind)
{
var tags = runtimeMetrics.DefaultTags
.Concat(new[]
{
new KeyValuePair<string, object?>("decision", allowed ? "allow" : "deny"),
new KeyValuePair<string, object?>("source", fromCache ? "cache" : "backend"),
new KeyValuePair<string, object?>("fail_open", failOpen ? "true" : "false"),
new KeyValuePair<string, object?>("event", eventKind.ToString())
})
.ToArray();
runtimeMetrics.AdmissionDecisions.Add(1, tags);
}
private bool ShouldFailOpen(ZastavaWebhookAdmissionOptions admission, string? @namespace)
{
if (@namespace is null)
{
return admission.FailOpenByDefault;
}
if (admission.FailClosedNamespaces.Contains(@namespace))
{
return false;
}
if (admission.FailOpenNamespaces.Contains(@namespace))
{
return true;
}
return admission.FailOpenByDefault;
}
private DateTimeOffset CalculateExpiry(RuntimePolicyResponse response)
{
var now = timeProvider.GetUtcNow();
var ttlSeconds = Math.Max(1, response.TtlSeconds);
var intended = now.AddSeconds(ttlSeconds);
if (response.ExpiresAtUtc != default)
{
return response.ExpiresAtUtc < intended ? response.ExpiresAtUtc : intended;
}
return intended;
}
private sealed record ResolvedDigest(string Digest, IReadOnlyList<ImageResolutionResult> Entries);
}
internal sealed record RuntimeAdmissionRequest(
string? Namespace,
IReadOnlyDictionary<string, string> Labels,
IReadOnlyList<string> Images);
internal sealed record RuntimeAdmissionDecision
{
public required string OriginalImage { get; init; }
public string? ResolvedDigest { get; init; }
public PolicyVerdict Verdict { get; init; }
public bool Allowed { get; init; }
public RuntimePolicyImageResult? Policy { get; init; }
public IReadOnlyList<string> Reasons { get; init; } = Array.Empty<string>();
public bool FromCache { get; init; }
public bool ResolutionFailed { get; init; }
}
internal sealed record RuntimeAdmissionEvaluation
{
public required IReadOnlyList<RuntimeAdmissionDecision> Decisions { get; init; }
public bool BackendFailed { get; init; }
public bool FailOpenApplied { get; init; }
public string? FailureReason { get; init; }
public int TtlSeconds { get; init; }
public static RuntimeAdmissionEvaluation Empty()
=> new()
{
Decisions = Array.Empty<RuntimeAdmissionDecision>(),
BackendFailed = false,
FailOpenApplied = false,
FailureReason = null,
TtlSeconds = 0
};
}

View File

@@ -0,0 +1,83 @@
using System.Collections.Concurrent;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Webhook.Backend;
using StellaOps.Zastava.Webhook.Configuration;
namespace StellaOps.Zastava.Webhook.Admission;
internal sealed class RuntimePolicyCache
{
private readonly ConcurrentDictionary<string, CacheEntry> entries = new(StringComparer.Ordinal);
private readonly ILogger<RuntimePolicyCache> logger;
private readonly TimeProvider timeProvider;
public RuntimePolicyCache(IOptions<ZastavaWebhookOptions> options, TimeProvider timeProvider, ILogger<RuntimePolicyCache> logger)
{
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
ArgumentNullException.ThrowIfNull(options);
var admission = options.Value.Admission;
if (!string.IsNullOrWhiteSpace(admission.CacheSeedPath) && File.Exists(admission.CacheSeedPath))
{
TryLoadSeed(admission.CacheSeedPath!);
}
}
public bool TryGet(string digest, out RuntimePolicyImageResult result)
{
if (entries.TryGetValue(digest, out var entry))
{
if (timeProvider.GetUtcNow() <= entry.ExpiresAtUtc)
{
result = entry.Result;
return true;
}
entries.TryRemove(digest, out _);
}
result = default!;
return false;
}
public void Set(string digest, RuntimePolicyImageResult result, DateTimeOffset expiresAtUtc)
{
entries[digest] = new CacheEntry(result, expiresAtUtc);
}
private void TryLoadSeed(string path)
{
try
{
var payload = File.ReadAllText(path);
var seed = JsonSerializer.Deserialize<RuntimePolicyResponse>(payload, new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
});
if (seed?.Results is null || seed.Results.Count == 0)
{
logger.LogDebug("Runtime policy cache seed file {Path} empty or invalid.", path);
return;
}
var ttlSeconds = Math.Max(1, seed.TtlSeconds);
var expires = timeProvider.GetUtcNow().AddSeconds(ttlSeconds);
foreach (var pair in seed.Results)
{
Set(pair.Key, pair.Value, expires);
}
logger.LogInformation("Loaded {Count} runtime policy cache seed entries from {Path}.", seed.Results.Count, path);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to load runtime policy cache seed from {Path}.", path);
}
}
private sealed record CacheEntry(RuntimePolicyImageResult Result, DateTimeOffset ExpiresAtUtc);
}

View File

@@ -10,6 +10,12 @@ public sealed record RuntimePolicyResponse
[JsonPropertyName("ttlSeconds")]
public int TtlSeconds { get; init; }
[JsonPropertyName("expiresAtUtc")]
public DateTimeOffset ExpiresAtUtc { get; init; }
[JsonPropertyName("policyRevision")]
public string? PolicyRevision { get; init; }
[JsonPropertyName("results")]
public IReadOnlyDictionary<string, RuntimePolicyImageResult> Results { get; init; } = new Dictionary<string, RuntimePolicyImageResult>();
}

View File

@@ -2,6 +2,7 @@ using System;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Core.Configuration;
using StellaOps.Zastava.Webhook.Admission;
using StellaOps.Zastava.Webhook.Authority;
using StellaOps.Zastava.Webhook.Backend;
using StellaOps.Zastava.Webhook.Certificates;
@@ -22,12 +23,19 @@ public static class ServiceCollectionExtensions
.ValidateDataAnnotations()
.ValidateOnStart();
services.TryAddSingleton(TimeProvider.System);
services.TryAddEnumerable(ServiceDescriptor.Singleton<IWebhookCertificateSource, SecretFileCertificateSource>());
services.TryAddEnumerable(ServiceDescriptor.Singleton<IWebhookCertificateSource, CsrCertificateSource>());
services.TryAddSingleton<IWebhookCertificateProvider, WebhookCertificateProvider>();
services.TryAddSingleton<WebhookCertificateHealthCheck>();
services.TryAddEnumerable(ServiceDescriptor.Singleton<IPostConfigureOptions<ZastavaRuntimeOptions>, WebhookRuntimeOptionsPostConfigure>());
services.TryAddSingleton<AdmissionReviewParser>();
services.TryAddSingleton<AdmissionResponseBuilder>();
services.TryAddSingleton<RuntimePolicyCache>();
services.TryAddSingleton<IImageDigestResolver, ImageDigestResolver>();
services.TryAddSingleton<IRuntimeAdmissionPolicyService, RuntimeAdmissionPolicyService>();
services.AddHttpClient<IRuntimePolicyClient, RuntimePolicyClient>((provider, client) =>
{
var backend = provider.GetRequiredService<IOptions<ZastavaWebhookOptions>>().Value.Backend;

View File

@@ -2,6 +2,7 @@ using System.Security.Authentication;
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
using Serilog;
using Serilog.Events;
using StellaOps.Zastava.Webhook.Admission;
using StellaOps.Zastava.Webhook.Authority;
using StellaOps.Zastava.Webhook.Certificates;
using StellaOps.Zastava.Webhook.Configuration;
@@ -59,9 +60,8 @@ app.MapHealthChecks("/healthz/live", new HealthCheckOptions
Predicate = _ => false
});
// Placeholder admission endpoint; will be replaced as tasks 12-102/12-103 land.
app.MapPost("/admission", () => Results.StatusCode(StatusCodes.Status501NotImplemented))
.WithName("AdmissionReview");
app.MapPost("/admission", AdmissionEndpoint.HandleAsync)
.WithName("AdmissionReview");
app.MapGet("/", () => Results.Ok(new { status = "ok", service = "zastava-webhook" }));

View File

@@ -3,8 +3,8 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| ZASTAVA-WEBHOOK-12-101 | DONE (2025-10-24) | Zastava Webhook Guild | — | Admission controller host with TLS bootstrap and Authority auth. | Webhook host boots with deterministic TLS bootstrap, enforces Authority-issued credentials, e2e smoke proves admission callback lifecycle, structured logs + metrics emit on each decision. |
| ZASTAVA-WEBHOOK-12-102 | DOING | Zastava Webhook Guild | — | Query Scanner `/policy/runtime`, resolve digests, enforce verdicts. | Scanner client resolves image digests + policy verdicts, unit tests cover allow/deny, integration harness rejects/admits workloads per policy with deterministic payloads. |
| ZASTAVA-WEBHOOK-12-103 | DOING | Zastava Webhook Guild | — | Caching, fail-open/closed toggles, metrics/logging for admission decisions. | Configurable cache TTL + seeds survive restart, fail-open/closed toggles verified via tests, metrics/logging exported per decision path, docs note operational knobs. |
| ZASTAVA-WEBHOOK-12-104 | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-102 | Wire `/admission` endpoint to runtime policy client and emit allow/deny envelopes. | Admission handler resolves pods to digests, invokes policy client, returns canonical `AdmissionDecisionEnvelope` with deterministic logging and metrics. |
| ZASTAVA-WEBHOOK-12-102 | DONE (2025-10-24) | Zastava Webhook Guild | — | Query Scanner `/policy/runtime`, resolve digests, enforce verdicts. | Scanner client resolves image digests + policy verdicts, unit tests cover allow/deny, integration harness rejects/admits workloads per policy with deterministic payloads. |
| ZASTAVA-WEBHOOK-12-103 | DONE (2025-10-24) | Zastava Webhook Guild | — | Caching, fail-open/closed toggles, metrics/logging for admission decisions. | Configurable cache TTL + seeds survive restart, fail-open/closed toggles verified via tests, metrics/logging exported per decision path, docs note operational knobs. |
| ZASTAVA-WEBHOOK-12-104 | DONE (2025-10-24) | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-102 | Wire `/admission` endpoint to runtime policy client and emit allow/deny envelopes. | Admission handler resolves pods to digests, invokes policy client, returns canonical `AdmissionDecisionEnvelope` with deterministic logging and metrics. |
> Status update · 2025-10-19: Confirmed no prerequisites for ZASTAVA-WEBHOOK-12-101/102/103; tasks moved to DOING for kickoff. Implementation plan covering TLS bootstrap, backend contract, caching/metrics recorded in `IMPLEMENTATION_PLAN.md`.

View File

@@ -337,6 +337,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Zastava.Observer", "StellaOps.Zastava.Observer\StellaOps.Zastava.Observer.csproj", "{BC38594B-0B84-4657-9F7B-F2A0FC810F04}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Zastava.Observer.Tests", "StellaOps.Zastava.Observer.Tests\StellaOps.Zastava.Observer.Tests.csproj", "{20E0774F-86D5-4CD0-B636-E5212074FDE8}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -2291,6 +2293,18 @@ Global
{BC38594B-0B84-4657-9F7B-F2A0FC810F04}.Release|x64.Build.0 = Release|Any CPU
{BC38594B-0B84-4657-9F7B-F2A0FC810F04}.Release|x86.ActiveCfg = Release|Any CPU
{BC38594B-0B84-4657-9F7B-F2A0FC810F04}.Release|x86.Build.0 = Release|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Debug|x64.ActiveCfg = Debug|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Debug|x64.Build.0 = Debug|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Debug|x86.ActiveCfg = Debug|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Debug|x86.Build.0 = Debug|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Release|Any CPU.Build.0 = Release|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Release|x64.ActiveCfg = Release|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Release|x64.Build.0 = Release|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Release|x86.ActiveCfg = Release|Any CPU
{20E0774F-86D5-4CD0-B636-E5212074FDE8}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE