up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Build Test Deploy / build-test (push) Has been cancelled
Build Test Deploy / authority-container (push) Has been cancelled
Build Test Deploy / docs (push) Has been cancelled
Build Test Deploy / deploy (push) Has been cancelled

This commit is contained in:
2025-10-19 10:38:55 +03:00
parent c4980d9625
commit daa6a4ae8c
250 changed files with 17967 additions and 66 deletions

View File

@@ -7,6 +7,8 @@
<IsConcelierPlugin Condition="'$(IsConcelierPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)').StartsWith('StellaOps.Concelier.Connector.'))">true</IsConcelierPlugin>
<IsConcelierPlugin Condition="'$(IsConcelierPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)').StartsWith('StellaOps.Concelier.Exporter.'))">true</IsConcelierPlugin>
<IsAuthorityPlugin Condition="'$(IsAuthorityPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)').StartsWith('StellaOps.Authority.Plugin.'))">true</IsAuthorityPlugin>
<ScannerBuildxPluginOutputRoot Condition="'$(ScannerBuildxPluginOutputRoot)' == ''">$([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)..\plugins\scanner\buildx\'))</ScannerBuildxPluginOutputRoot>
<IsScannerBuildxPlugin Condition="'$(IsScannerBuildxPlugin)' == '' and $([System.String]::Copy('$(MSBuildProjectName)')) == 'StellaOps.Scanner.Sbomer.BuildXPlugin'">true</IsScannerBuildxPlugin>
</PropertyGroup>
<ItemGroup>

View File

@@ -30,4 +30,21 @@
<Copy SourceFiles="@(AuthorityPluginArtifacts)" DestinationFolder="$(AuthorityPluginOutputDirectory)" SkipUnchangedFiles="true" />
</Target>
<Target Name="ScannerCopyBuildxPluginArtifacts" AfterTargets="Build" Condition="'$(IsScannerBuildxPlugin)' == 'true'">
<PropertyGroup>
<ScannerBuildxPluginOutputDirectory>$(ScannerBuildxPluginOutputRoot)\$(MSBuildProjectName)</ScannerBuildxPluginOutputDirectory>
</PropertyGroup>
<MakeDir Directories="$(ScannerBuildxPluginOutputDirectory)" />
<ItemGroup>
<ScannerBuildxPluginArtifacts Include="$(TargetPath)" />
<ScannerBuildxPluginArtifacts Include="$(TargetPath).deps.json" Condition="Exists('$(TargetPath).deps.json')" />
<ScannerBuildxPluginArtifacts Include="$(TargetDir)$(TargetName).pdb" Condition="Exists('$(TargetDir)$(TargetName).pdb')" />
<ScannerBuildxPluginArtifacts Include="$(ProjectDir)stellaops.sbom-indexer.manifest.json" Condition="Exists('$(ProjectDir)stellaops.sbom-indexer.manifest.json')" />
</ItemGroup>
<Copy SourceFiles="@(ScannerBuildxPluginArtifacts)" DestinationFolder="$(ScannerBuildxPluginOutputDirectory)" SkipUnchangedFiles="true" />
</Target>
</Project>

View File

@@ -0,0 +1,86 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
namespace StellaOps.Policy.Tests;
public sealed class PolicyBinderTests
{
[Fact]
public void Bind_ValidYaml_ReturnsSuccess()
{
const string yaml = """
version: "1.0"
rules:
- name: Block Critical
severity: [Critical]
sources: [NVD]
action: block
""";
var result = PolicyBinder.Bind(yaml, PolicyDocumentFormat.Yaml);
Assert.True(result.Success);
Assert.Equal("1.0", result.Document.Version);
Assert.Single(result.Document.Rules);
Assert.Empty(result.Issues);
}
[Fact]
public void Bind_InvalidSeverity_ReturnsError()
{
const string yaml = """
version: "1.0"
rules:
- name: Invalid Severity
severity: [Nope]
action: block
""";
var result = PolicyBinder.Bind(yaml, PolicyDocumentFormat.Yaml);
Assert.False(result.Success);
Assert.Contains(result.Issues, issue => issue.Code == "policy.severity.invalid");
}
[Fact]
public async Task Cli_StrictMode_FailsOnWarnings()
{
const string yaml = """
version: "1.0"
rules:
- name: Quiet Warning
sources: ["", "NVD"]
action: ignore
""";
var path = Path.Combine(Path.GetTempPath(), $"policy-{Guid.NewGuid():N}.yaml");
await File.WriteAllTextAsync(path, yaml);
try
{
using var output = new StringWriter();
using var error = new StringWriter();
var cli = new PolicyValidationCli(output, error);
var options = new PolicyValidationCliOptions
{
Inputs = new[] { path },
Strict = true,
};
var exitCode = await cli.RunAsync(options, CancellationToken.None);
Assert.Equal(2, exitCode);
Assert.Contains("WARNING", output.ToString());
}
finally
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
}

View File

@@ -0,0 +1,166 @@
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using Xunit;
namespace StellaOps.Policy.Tests;
public sealed class PolicyPreviewServiceTests
{
[Fact]
public async Task PreviewAsync_ComputesDiffs_ForBlockingRule()
{
const string yaml = """
version: "1.0"
rules:
- name: Block Critical
severity: [Critical]
action: block
""";
var snapshotRepo = new InMemoryPolicySnapshotRepository();
var auditRepo = new InMemoryPolicyAuditRepository();
var timeProvider = new FakeTimeProvider();
var store = new PolicySnapshotStore(snapshotRepo, auditRepo, timeProvider, NullLogger<PolicySnapshotStore>.Instance);
await store.SaveAsync(new PolicySnapshotContent(yaml, PolicyDocumentFormat.Yaml, "tester", null, null), CancellationToken.None);
var service = new PolicyPreviewService(store, NullLogger<PolicyPreviewService>.Instance);
var findings = ImmutableArray.Create(
PolicyFinding.Create("finding-1", PolicySeverity.Critical, environment: "prod", source: "NVD"),
PolicyFinding.Create("finding-2", PolicySeverity.Low));
var baseline = ImmutableArray.Create(
new PolicyVerdict("finding-1", PolicyVerdictStatus.Pass),
new PolicyVerdict("finding-2", PolicyVerdictStatus.Pass));
var response = await service.PreviewAsync(new PolicyPreviewRequest(
"sha256:abc",
findings,
baseline),
CancellationToken.None);
Assert.True(response.Success);
Assert.Equal(1, response.ChangedCount);
var diff1 = Assert.Single(response.Diffs.Where(diff => diff.Projected.FindingId == "finding-1"));
Assert.Equal(PolicyVerdictStatus.Pass, diff1.Baseline.Status);
Assert.Equal(PolicyVerdictStatus.Blocked, diff1.Projected.Status);
Assert.Equal("Block Critical", diff1.Projected.RuleName);
Assert.True(diff1.Projected.Score > 0);
Assert.Equal(PolicyScoringConfig.Default.Version, diff1.Projected.ConfigVersion);
Assert.Equal(PolicyVerdictStatus.Pass, response.Diffs.First(diff => diff.Projected.FindingId == "finding-2").Projected.Status);
}
[Fact]
public async Task PreviewAsync_UsesProposedPolicy_WhenProvided()
{
const string yaml = """
version: "1.0"
rules:
- name: Ignore Dev
environments: [dev]
action:
type: ignore
justification: dev waiver
""";
var snapshotRepo = new InMemoryPolicySnapshotRepository();
var auditRepo = new InMemoryPolicyAuditRepository();
var store = new PolicySnapshotStore(snapshotRepo, auditRepo, TimeProvider.System, NullLogger<PolicySnapshotStore>.Instance);
var service = new PolicyPreviewService(store, NullLogger<PolicyPreviewService>.Instance);
var findings = ImmutableArray.Create(
PolicyFinding.Create("finding-1", PolicySeverity.Medium, environment: "dev"));
var baseline = ImmutableArray.Create(new PolicyVerdict("finding-1", PolicyVerdictStatus.Blocked));
var response = await service.PreviewAsync(new PolicyPreviewRequest(
"sha256:def",
findings,
baseline,
SnapshotOverride: null,
ProposedPolicy: new PolicySnapshotContent(yaml, PolicyDocumentFormat.Yaml, "tester", null, "dev override")),
CancellationToken.None);
Assert.True(response.Success);
var diff = Assert.Single(response.Diffs);
Assert.Equal(PolicyVerdictStatus.Blocked, diff.Baseline.Status);
Assert.Equal(PolicyVerdictStatus.Ignored, diff.Projected.Status);
Assert.Equal("Ignore Dev", diff.Projected.RuleName);
Assert.True(diff.Projected.Score >= 0);
Assert.Equal(1, response.ChangedCount);
}
[Fact]
public async Task PreviewAsync_ReturnsIssues_WhenPolicyInvalid()
{
var snapshotRepo = new InMemoryPolicySnapshotRepository();
var auditRepo = new InMemoryPolicyAuditRepository();
var store = new PolicySnapshotStore(snapshotRepo, auditRepo, TimeProvider.System, NullLogger<PolicySnapshotStore>.Instance);
var service = new PolicyPreviewService(store, NullLogger<PolicyPreviewService>.Instance);
const string invalid = "version: 1.0";
var request = new PolicyPreviewRequest(
"sha256:ghi",
ImmutableArray<PolicyFinding>.Empty,
ImmutableArray<PolicyVerdict>.Empty,
SnapshotOverride: null,
ProposedPolicy: new PolicySnapshotContent(invalid, PolicyDocumentFormat.Yaml, null, null, null));
var response = await service.PreviewAsync(request, CancellationToken.None);
Assert.False(response.Success);
Assert.NotEmpty(response.Issues);
}
[Fact]
public async Task PreviewAsync_QuietWithoutVexDowngradesToWarn()
{
const string yaml = """
version: "1.0"
rules:
- name: Quiet Without VEX
severity: [Low]
quiet: true
action:
type: ignore
""";
var binding = PolicyBinder.Bind(yaml, PolicyDocumentFormat.Yaml);
Assert.True(binding.Success);
Assert.Empty(binding.Issues);
Assert.False(binding.Document.Rules[0].Metadata.ContainsKey("quiet"));
Assert.True(binding.Document.Rules[0].Action.Quiet);
var store = new PolicySnapshotStore(new InMemoryPolicySnapshotRepository(), new InMemoryPolicyAuditRepository(), TimeProvider.System, NullLogger<PolicySnapshotStore>.Instance);
await store.SaveAsync(new PolicySnapshotContent(yaml, PolicyDocumentFormat.Yaml, "tester", null, "quiet test"), CancellationToken.None);
var snapshot = await store.GetLatestAsync();
Assert.NotNull(snapshot);
Assert.True(snapshot!.Document.Rules[0].Action.Quiet);
Assert.Null(snapshot.Document.Rules[0].Action.RequireVex);
Assert.Equal(PolicyActionType.Ignore, snapshot.Document.Rules[0].Action.Type);
var manualVerdict = PolicyEvaluation.EvaluateFinding(snapshot.Document, snapshot.ScoringConfig, PolicyFinding.Create("finding-quiet", PolicySeverity.Low));
Assert.Equal(PolicyVerdictStatus.Warned, manualVerdict.Status);
var service = new PolicyPreviewService(store, NullLogger<PolicyPreviewService>.Instance);
var findings = ImmutableArray.Create(PolicyFinding.Create("finding-quiet", PolicySeverity.Low));
var baseline = ImmutableArray<PolicyVerdict>.Empty;
var response = await service.PreviewAsync(new PolicyPreviewRequest(
"sha256:quiet",
findings,
baseline),
CancellationToken.None);
Assert.True(response.Success);
var verdict = Assert.Single(response.Diffs).Projected;
Assert.Equal(PolicyVerdictStatus.Warned, verdict.Status);
Assert.Contains("requireVex", verdict.Notes, System.StringComparison.OrdinalIgnoreCase);
Assert.True(verdict.Score >= 0);
}
}

View File

@@ -0,0 +1,26 @@
using System.Threading.Tasks;
using Xunit;
namespace StellaOps.Policy.Tests;
public sealed class PolicyScoringConfigTests
{
[Fact]
public void LoadDefaultReturnsConfig()
{
var config = PolicyScoringConfigBinder.LoadDefault();
Assert.NotNull(config);
Assert.Equal("1.0", config.Version);
Assert.NotEmpty(config.SeverityWeights);
Assert.True(config.SeverityWeights.ContainsKey(PolicySeverity.Critical));
Assert.True(config.QuietPenalty > 0);
}
[Fact]
public void BindRejectsEmptyContent()
{
var result = PolicyScoringConfigBinder.Bind(string.Empty, PolicyDocumentFormat.Json);
Assert.False(result.Success);
Assert.NotEmpty(result.Issues);
}
}

View File

@@ -0,0 +1,94 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using Xunit;
namespace StellaOps.Policy.Tests;
public sealed class PolicySnapshotStoreTests
{
private const string BasePolicyYaml = """
version: "1.0"
rules:
- name: Block Critical
severity: [Critical]
action: block
""";
[Fact]
public async Task SaveAsync_CreatesNewSnapshotAndAuditEntry()
{
var snapshotRepo = new InMemoryPolicySnapshotRepository();
var auditRepo = new InMemoryPolicyAuditRepository();
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 10, 0, 0, TimeSpan.Zero));
var store = new PolicySnapshotStore(snapshotRepo, auditRepo, timeProvider, NullLogger<PolicySnapshotStore>.Instance);
var content = new PolicySnapshotContent(BasePolicyYaml, PolicyDocumentFormat.Yaml, "cli", "test", null);
var result = await store.SaveAsync(content, CancellationToken.None);
Assert.True(result.Success);
Assert.True(result.Created);
Assert.NotNull(result.Snapshot);
Assert.Equal("rev-1", result.Snapshot!.RevisionId);
Assert.Equal(result.Digest, result.Snapshot.Digest);
Assert.Equal(timeProvider.GetUtcNow(), result.Snapshot.CreatedAt);
Assert.Equal(PolicyScoringConfig.Default.Version, result.Snapshot.ScoringConfig.Version);
var latest = await store.GetLatestAsync();
Assert.Equal(result.Snapshot, latest);
var audits = await auditRepo.ListAsync(10);
Assert.Single(audits);
Assert.Equal(result.Digest, audits[0].Digest);
Assert.Equal("snapshot.created", audits[0].Action);
Assert.Equal("rev-1", audits[0].RevisionId);
}
[Fact]
public async Task SaveAsync_DoesNotCreateNewRevisionWhenDigestUnchanged()
{
var snapshotRepo = new InMemoryPolicySnapshotRepository();
var auditRepo = new InMemoryPolicyAuditRepository();
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 10, 0, 0, TimeSpan.Zero));
var store = new PolicySnapshotStore(snapshotRepo, auditRepo, timeProvider, NullLogger<PolicySnapshotStore>.Instance);
var content = new PolicySnapshotContent(BasePolicyYaml, PolicyDocumentFormat.Yaml, "cli", "test", null);
var first = await store.SaveAsync(content, CancellationToken.None);
Assert.True(first.Created);
timeProvider.Advance(TimeSpan.FromHours(1));
var second = await store.SaveAsync(content, CancellationToken.None);
Assert.True(second.Success);
Assert.False(second.Created);
Assert.Equal(first.Digest, second.Digest);
Assert.Equal("rev-1", second.Snapshot!.RevisionId);
Assert.Equal(PolicyScoringConfig.Default.Version, second.Snapshot.ScoringConfig.Version);
var audits = await auditRepo.ListAsync(10);
Assert.Single(audits);
}
[Fact]
public async Task SaveAsync_ReturnsFailureWhenValidationFails()
{
var snapshotRepo = new InMemoryPolicySnapshotRepository();
var auditRepo = new InMemoryPolicyAuditRepository();
var store = new PolicySnapshotStore(snapshotRepo, auditRepo, TimeProvider.System, NullLogger<PolicySnapshotStore>.Instance);
const string invalidYaml = "version: '1.0'\nrules: []";
var content = new PolicySnapshotContent(invalidYaml, PolicyDocumentFormat.Yaml, null, null, null);
var result = await store.SaveAsync(content, CancellationToken.None);
Assert.False(result.Success);
Assert.False(result.Created);
Assert.Null(result.Snapshot);
var audits = await auditRepo.ListAsync(5);
Assert.Empty(audits);
}
}

View File

@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Policy\StellaOps.Policy.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,12 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Policy;
public interface IPolicyAuditRepository
{
Task AddAsync(PolicyAuditEntry entry, CancellationToken cancellationToken = default);
Task<IReadOnlyList<PolicyAuditEntry>> ListAsync(int limit, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Policy;
public sealed class InMemoryPolicyAuditRepository : IPolicyAuditRepository
{
private readonly List<PolicyAuditEntry> _entries = new();
private readonly SemaphoreSlim _mutex = new(1, 1);
public async Task AddAsync(PolicyAuditEntry entry, CancellationToken cancellationToken = default)
{
if (entry is null)
{
throw new ArgumentNullException(nameof(entry));
}
await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_entries.Add(entry);
_entries.Sort(static (left, right) => left.CreatedAt.CompareTo(right.CreatedAt));
}
finally
{
_mutex.Release();
}
}
public async Task<IReadOnlyList<PolicyAuditEntry>> ListAsync(int limit, CancellationToken cancellationToken = default)
{
await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
IEnumerable<PolicyAuditEntry> query = _entries;
if (limit > 0)
{
query = query.TakeLast(limit);
}
return query.ToImmutableArray();
}
finally
{
_mutex.Release();
}
}
}

View File

@@ -0,0 +1,12 @@
using System;
namespace StellaOps.Policy;
public sealed record PolicyAuditEntry(
Guid Id,
DateTimeOffset CreatedAt,
string Action,
string RevisionId,
string Digest,
string? Actor,
string Message);

View File

@@ -0,0 +1,913 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace StellaOps.Policy;
public enum PolicyDocumentFormat
{
Json,
Yaml,
}
public sealed record PolicyBindingResult(
bool Success,
PolicyDocument Document,
ImmutableArray<PolicyIssue> Issues,
PolicyDocumentFormat Format);
public static class PolicyBinder
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString,
Converters =
{
new JsonStringEnumConverter()
},
};
private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.IgnoreUnmatchedProperties()
.Build();
public static PolicyBindingResult Bind(string content, PolicyDocumentFormat format)
{
if (string.IsNullOrWhiteSpace(content))
{
var issues = ImmutableArray.Create(
PolicyIssue.Error("policy.empty", "Policy document is empty.", "$"));
return new PolicyBindingResult(false, PolicyDocument.Empty, issues, format);
}
try
{
var node = ParseToNode(content, format);
if (node is not JsonObject obj)
{
var issues = ImmutableArray.Create(
PolicyIssue.Error("policy.document.invalid", "Policy document must be an object.", "$"));
return new PolicyBindingResult(false, PolicyDocument.Empty, issues, format);
}
var model = obj.Deserialize<PolicyDocumentModel>(SerializerOptions) ?? new PolicyDocumentModel();
var normalization = PolicyNormalizer.Normalize(model);
var success = normalization.Issues.All(static issue => issue.Severity != PolicyIssueSeverity.Error);
return new PolicyBindingResult(success, normalization.Document, normalization.Issues, format);
}
catch (JsonException ex)
{
var issues = ImmutableArray.Create(
PolicyIssue.Error("policy.parse.json", $"Failed to parse policy JSON: {ex.Message}", "$"));
return new PolicyBindingResult(false, PolicyDocument.Empty, issues, format);
}
catch (YamlDotNet.Core.YamlException ex)
{
var issues = ImmutableArray.Create(
PolicyIssue.Error("policy.parse.yaml", $"Failed to parse policy YAML: {ex.Message}", "$"));
return new PolicyBindingResult(false, PolicyDocument.Empty, issues, format);
}
}
public static PolicyBindingResult Bind(Stream stream, PolicyDocumentFormat format, Encoding? encoding = null)
{
if (stream is null)
{
throw new ArgumentNullException(nameof(stream));
}
encoding ??= Encoding.UTF8;
using var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true, leaveOpen: true);
var content = reader.ReadToEnd();
return Bind(content, format);
}
private static JsonNode? ParseToNode(string content, PolicyDocumentFormat format)
{
return format switch
{
PolicyDocumentFormat.Json => JsonNode.Parse(content, documentOptions: new JsonDocumentOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip,
}),
PolicyDocumentFormat.Yaml => ConvertYamlToJsonNode(content),
_ => throw new ArgumentOutOfRangeException(nameof(format), format, "Unsupported policy document format."),
};
}
private static JsonNode? ConvertYamlToJsonNode(string content)
{
var yamlObject = YamlDeserializer.Deserialize<object?>(content);
return ConvertYamlObject(yamlObject);
}
private static JsonNode? ConvertYamlObject(object? value)
{
switch (value)
{
case null:
return null;
case string s:
return JsonValue.Create(s);
case bool b:
return JsonValue.Create(b);
case sbyte or byte or short or ushort or int or uint or long or ulong or float or double or decimal:
return JsonValue.Create(Convert.ToDecimal(value, CultureInfo.InvariantCulture));
case DateTime dt:
return JsonValue.Create(dt.ToString("O", CultureInfo.InvariantCulture));
case DateTimeOffset dto:
return JsonValue.Create(dto.ToString("O", CultureInfo.InvariantCulture));
case Enum e:
return JsonValue.Create(e.ToString());
case IDictionary dictionary:
{
var obj = new JsonObject();
foreach (DictionaryEntry entry in dictionary)
{
if (entry.Key is null)
{
continue;
}
var key = Convert.ToString(entry.Key, CultureInfo.InvariantCulture);
if (string.IsNullOrWhiteSpace(key))
{
continue;
}
obj[key!] = ConvertYamlObject(entry.Value);
}
return obj;
}
case IEnumerable enumerable:
{
var array = new JsonArray();
foreach (var item in enumerable)
{
array.Add(ConvertYamlObject(item));
}
return array;
}
default:
return JsonValue.Create(value.ToString());
}
}
private sealed record PolicyDocumentModel
{
[JsonPropertyName("version")]
public JsonNode? Version { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("metadata")]
public Dictionary<string, JsonNode?>? Metadata { get; init; }
[JsonPropertyName("rules")]
public List<PolicyRuleModel>? Rules { get; init; }
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extensions { get; init; }
}
private sealed record PolicyRuleModel
{
[JsonPropertyName("id")]
public string? Identifier { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("description")]
public string? Description { get; init; }
[JsonPropertyName("severity")]
public List<string>? Severity { get; init; }
[JsonPropertyName("sources")]
public List<string>? Sources { get; init; }
[JsonPropertyName("vendors")]
public List<string>? Vendors { get; init; }
[JsonPropertyName("licenses")]
public List<string>? Licenses { get; init; }
[JsonPropertyName("tags")]
public List<string>? Tags { get; init; }
[JsonPropertyName("environments")]
public List<string>? Environments { get; init; }
[JsonPropertyName("images")]
public List<string>? Images { get; init; }
[JsonPropertyName("repositories")]
public List<string>? Repositories { get; init; }
[JsonPropertyName("packages")]
public List<string>? Packages { get; init; }
[JsonPropertyName("purls")]
public List<string>? Purls { get; init; }
[JsonPropertyName("cves")]
public List<string>? Cves { get; init; }
[JsonPropertyName("paths")]
public List<string>? Paths { get; init; }
[JsonPropertyName("layerDigests")]
public List<string>? LayerDigests { get; init; }
[JsonPropertyName("usedByEntrypoint")]
public List<string>? UsedByEntrypoint { get; init; }
[JsonPropertyName("action")]
public JsonNode? Action { get; init; }
[JsonPropertyName("expires")]
public JsonNode? Expires { get; init; }
[JsonPropertyName("until")]
public JsonNode? Until { get; init; }
[JsonPropertyName("justification")]
public string? Justification { get; init; }
[JsonPropertyName("quiet")]
public bool? Quiet { get; init; }
[JsonPropertyName("metadata")]
public Dictionary<string, JsonNode?>? Metadata { get; init; }
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extensions { get; init; }
}
private sealed class PolicyNormalizer
{
private static readonly ImmutableDictionary<string, PolicySeverity> SeverityMap =
new Dictionary<string, PolicySeverity>(StringComparer.OrdinalIgnoreCase)
{
["critical"] = PolicySeverity.Critical,
["high"] = PolicySeverity.High,
["medium"] = PolicySeverity.Medium,
["moderate"] = PolicySeverity.Medium,
["low"] = PolicySeverity.Low,
["informational"] = PolicySeverity.Informational,
["info"] = PolicySeverity.Informational,
["none"] = PolicySeverity.None,
["unknown"] = PolicySeverity.Unknown,
}.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase);
public static (PolicyDocument Document, ImmutableArray<PolicyIssue> Issues) Normalize(PolicyDocumentModel model)
{
var issues = ImmutableArray.CreateBuilder<PolicyIssue>();
var version = NormalizeVersion(model.Version, issues);
var metadata = NormalizeMetadata(model.Metadata, "$.metadata", issues);
var rules = NormalizeRules(model.Rules, issues);
if (model.Extensions is { Count: > 0 })
{
foreach (var pair in model.Extensions)
{
issues.Add(PolicyIssue.Warning(
"policy.document.extension",
$"Unrecognized document property '{pair.Key}' has been ignored.",
$"$.{pair.Key}"));
}
}
var document = new PolicyDocument(
version ?? PolicySchema.CurrentVersion,
rules,
metadata);
var orderedIssues = SortIssues(issues);
return (document, orderedIssues);
}
private static string? NormalizeVersion(JsonNode? versionNode, ImmutableArray<PolicyIssue>.Builder issues)
{
if (versionNode is null)
{
issues.Add(PolicyIssue.Warning("policy.version.missing", "Policy version not specified; defaulting to 1.0.", "$.version"));
return PolicySchema.CurrentVersion;
}
if (versionNode is JsonValue value)
{
if (value.TryGetValue(out string? versionText))
{
versionText = versionText?.Trim();
if (string.IsNullOrEmpty(versionText))
{
issues.Add(PolicyIssue.Error("policy.version.empty", "Policy version is empty.", "$.version"));
return null;
}
if (IsSupportedVersion(versionText))
{
return CanonicalizeVersion(versionText);
}
issues.Add(PolicyIssue.Error("policy.version.unsupported", $"Unsupported policy version '{versionText}'. Expected '{PolicySchema.CurrentVersion}'.", "$.version"));
return null;
}
if (value.TryGetValue(out double numericVersion))
{
var numericText = numericVersion.ToString("0.0###", CultureInfo.InvariantCulture);
if (IsSupportedVersion(numericText))
{
return CanonicalizeVersion(numericText);
}
issues.Add(PolicyIssue.Error("policy.version.unsupported", $"Unsupported policy version '{numericText}'.", "$.version"));
return null;
}
}
var raw = versionNode.ToJsonString();
issues.Add(PolicyIssue.Error("policy.version.invalid", $"Policy version must be a string. Received: {raw}", "$.version"));
return null;
}
private static bool IsSupportedVersion(string versionText)
=> string.Equals(versionText, "1", StringComparison.OrdinalIgnoreCase)
|| string.Equals(versionText, "1.0", StringComparison.OrdinalIgnoreCase)
|| string.Equals(versionText, PolicySchema.CurrentVersion, StringComparison.OrdinalIgnoreCase);
private static string CanonicalizeVersion(string versionText)
=> string.Equals(versionText, "1", StringComparison.OrdinalIgnoreCase)
? "1.0"
: versionText;
private static ImmutableDictionary<string, string> NormalizeMetadata(
Dictionary<string, JsonNode?>? metadata,
string path,
ImmutableArray<PolicyIssue>.Builder issues)
{
if (metadata is null || metadata.Count == 0)
{
return ImmutableDictionary<string, string>.Empty;
}
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
foreach (var pair in metadata)
{
var key = pair.Key?.Trim();
if (string.IsNullOrEmpty(key))
{
issues.Add(PolicyIssue.Warning("policy.metadata.key.empty", "Metadata keys must be non-empty strings.", path));
continue;
}
var value = ConvertNodeToString(pair.Value);
builder[key] = value;
}
return builder.ToImmutable();
}
private static ImmutableArray<PolicyRule> NormalizeRules(
List<PolicyRuleModel>? rules,
ImmutableArray<PolicyIssue>.Builder issues)
{
if (rules is null || rules.Count == 0)
{
issues.Add(PolicyIssue.Error("policy.rules.empty", "At least one rule must be defined.", "$.rules"));
return ImmutableArray<PolicyRule>.Empty;
}
var normalized = new List<(PolicyRule Rule, int Index)>(rules.Count);
var seenNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
for (var index = 0; index < rules.Count; index++)
{
var model = rules[index];
var normalizedRule = NormalizeRule(model, index, issues);
if (normalizedRule is null)
{
continue;
}
if (!seenNames.Add(normalizedRule.Name))
{
issues.Add(PolicyIssue.Warning(
"policy.rules.duplicateName",
$"Duplicate rule name '{normalizedRule.Name}' detected; evaluation order may be ambiguous.",
$"$.rules[{index}].name"));
}
normalized.Add((normalizedRule, index));
}
return normalized
.OrderBy(static tuple => tuple.Rule.Name, StringComparer.OrdinalIgnoreCase)
.ThenBy(static tuple => tuple.Rule.Identifier ?? string.Empty, StringComparer.OrdinalIgnoreCase)
.ThenBy(static tuple => tuple.Index)
.Select(static tuple => tuple.Rule)
.ToImmutableArray();
}
private static PolicyRule? NormalizeRule(
PolicyRuleModel model,
int index,
ImmutableArray<PolicyIssue>.Builder issues)
{
var basePath = $"$.rules[{index}]";
var name = NormalizeRequiredString(model.Name, $"{basePath}.name", "Rule name", issues);
if (name is null)
{
return null;
}
var identifier = NormalizeOptionalString(model.Identifier);
var description = NormalizeOptionalString(model.Description);
var metadata = NormalizeMetadata(model.Metadata, $"{basePath}.metadata", issues);
var severities = NormalizeSeverityList(model.Severity, $"{basePath}.severity", issues);
var environments = NormalizeStringList(model.Environments, $"{basePath}.environments", issues);
var sources = NormalizeStringList(model.Sources, $"{basePath}.sources", issues);
var vendors = NormalizeStringList(model.Vendors, $"{basePath}.vendors", issues);
var licenses = NormalizeStringList(model.Licenses, $"{basePath}.licenses", issues);
var tags = NormalizeStringList(model.Tags, $"{basePath}.tags", issues);
var match = new PolicyRuleMatchCriteria(
NormalizeStringList(model.Images, $"{basePath}.images", issues),
NormalizeStringList(model.Repositories, $"{basePath}.repositories", issues),
NormalizeStringList(model.Packages, $"{basePath}.packages", issues),
NormalizeStringList(model.Purls, $"{basePath}.purls", issues),
NormalizeStringList(model.Cves, $"{basePath}.cves", issues),
NormalizeStringList(model.Paths, $"{basePath}.paths", issues),
NormalizeStringList(model.LayerDigests, $"{basePath}.layerDigests", issues),
NormalizeStringList(model.UsedByEntrypoint, $"{basePath}.usedByEntrypoint", issues));
var action = NormalizeAction(model, basePath, issues);
var justification = NormalizeOptionalString(model.Justification);
var expires = NormalizeTemporal(model.Expires ?? model.Until, $"{basePath}.expires", issues);
if (model.Extensions is { Count: > 0 })
{
foreach (var pair in model.Extensions)
{
issues.Add(PolicyIssue.Warning(
"policy.rule.extension",
$"Unrecognized rule property '{pair.Key}' has been ignored.",
$"{basePath}.{pair.Key}"));
}
}
return PolicyRule.Create(
name,
action,
severities,
environments,
sources,
vendors,
licenses,
tags,
match,
expires,
justification,
identifier,
description,
metadata);
}
private static PolicyAction NormalizeAction(
PolicyRuleModel model,
string basePath,
ImmutableArray<PolicyIssue>.Builder issues)
{
var actionNode = model.Action;
var quiet = model.Quiet ?? false;
if (!quiet && model.Extensions is not null && model.Extensions.TryGetValue("quiet", out var quietExtension) && quietExtension.ValueKind == JsonValueKind.True)
{
quiet = true;
}
string? justification = NormalizeOptionalString(model.Justification);
DateTimeOffset? until = NormalizeTemporal(model.Until, $"{basePath}.until", issues);
DateTimeOffset? expires = NormalizeTemporal(model.Expires, $"{basePath}.expires", issues);
var effectiveUntil = until ?? expires;
if (actionNode is null)
{
issues.Add(PolicyIssue.Error("policy.action.missing", "Rule action is required.", $"{basePath}.action"));
return new PolicyAction(PolicyActionType.Block, null, null, null, Quiet: false);
}
string? actionType = null;
JsonObject? actionObject = null;
switch (actionNode)
{
case JsonValue value when value.TryGetValue(out string? text):
actionType = text;
break;
case JsonValue value when value.TryGetValue(out bool booleanValue):
actionType = booleanValue ? "block" : "ignore";
break;
case JsonObject obj:
actionObject = obj;
if (obj.TryGetPropertyValue("type", out var typeNode) && typeNode is JsonValue typeValue && typeValue.TryGetValue(out string? typeText))
{
actionType = typeText;
}
else
{
issues.Add(PolicyIssue.Error("policy.action.type", "Action object must contain a 'type' property.", $"{basePath}.action.type"));
}
if (obj.TryGetPropertyValue("quiet", out var quietNode) && quietNode is JsonValue quietValue && quietValue.TryGetValue(out bool quietFlag))
{
quiet = quietFlag;
}
if (obj.TryGetPropertyValue("until", out var untilNode))
{
effectiveUntil ??= NormalizeTemporal(untilNode, $"{basePath}.action.until", issues);
}
if (obj.TryGetPropertyValue("justification", out var justificationNode) && justificationNode is JsonValue justificationValue && justificationValue.TryGetValue(out string? justificationText))
{
justification = NormalizeOptionalString(justificationText);
}
break;
default:
actionType = actionNode.ToString();
break;
}
if (string.IsNullOrWhiteSpace(actionType))
{
issues.Add(PolicyIssue.Error("policy.action.type", "Action type is required.", $"{basePath}.action"));
return new PolicyAction(PolicyActionType.Block, null, null, null, Quiet: quiet);
}
actionType = actionType.Trim();
var (type, typeIssues) = MapActionType(actionType, $"{basePath}.action");
foreach (var issue in typeIssues)
{
issues.Add(issue);
}
PolicyIgnoreOptions? ignoreOptions = null;
PolicyEscalateOptions? escalateOptions = null;
PolicyRequireVexOptions? requireVexOptions = null;
if (type == PolicyActionType.Ignore)
{
ignoreOptions = new PolicyIgnoreOptions(effectiveUntil, justification);
}
else if (type == PolicyActionType.Escalate)
{
escalateOptions = NormalizeEscalateOptions(actionObject, $"{basePath}.action", issues);
}
else if (type == PolicyActionType.RequireVex)
{
requireVexOptions = NormalizeRequireVexOptions(actionObject, $"{basePath}.action", issues);
}
return new PolicyAction(type, ignoreOptions, escalateOptions, requireVexOptions, quiet);
}
private static (PolicyActionType Type, ImmutableArray<PolicyIssue> Issues) MapActionType(string value, string path)
{
var issues = ImmutableArray<PolicyIssue>.Empty;
var lower = value.ToLowerInvariant();
return lower switch
{
"block" or "fail" or "deny" => (PolicyActionType.Block, issues),
"ignore" or "mute" => (PolicyActionType.Ignore, issues),
"warn" or "warning" => (PolicyActionType.Warn, issues),
"defer" => (PolicyActionType.Defer, issues),
"escalate" => (PolicyActionType.Escalate, issues),
"requirevex" or "require_vex" or "require-vex" => (PolicyActionType.RequireVex, issues),
_ => (PolicyActionType.Block, ImmutableArray.Create(PolicyIssue.Warning(
"policy.action.unknown",
$"Unknown action '{value}' encountered. Defaulting to 'block'.",
path))),
};
}
private static PolicyEscalateOptions? NormalizeEscalateOptions(
JsonObject? actionObject,
string path,
ImmutableArray<PolicyIssue>.Builder issues)
{
if (actionObject is null)
{
return null;
}
PolicySeverity? minSeverity = null;
bool requireKev = false;
double? minEpss = null;
if (actionObject.TryGetPropertyValue("severity", out var severityNode) && severityNode is JsonValue severityValue && severityValue.TryGetValue(out string? severityText))
{
if (SeverityMap.TryGetValue(severityText ?? string.Empty, out var mapped))
{
minSeverity = mapped;
}
else
{
issues.Add(PolicyIssue.Warning("policy.action.escalate.severity", $"Unknown escalate severity '{severityText}'.", $"{path}.severity"));
}
}
if (actionObject.TryGetPropertyValue("kev", out var kevNode) && kevNode is JsonValue kevValue && kevValue.TryGetValue(out bool kevFlag))
{
requireKev = kevFlag;
}
if (actionObject.TryGetPropertyValue("epss", out var epssNode))
{
var parsed = ParseDouble(epssNode, $"{path}.epss", issues);
if (parsed is { } epssValue)
{
if (epssValue < 0 || epssValue > 1)
{
issues.Add(PolicyIssue.Warning("policy.action.escalate.epssRange", "EPS score must be between 0 and 1.", $"{path}.epss"));
}
else
{
minEpss = epssValue;
}
}
}
return new PolicyEscalateOptions(minSeverity, requireKev, minEpss);
}
private static PolicyRequireVexOptions? NormalizeRequireVexOptions(
JsonObject? actionObject,
string path,
ImmutableArray<PolicyIssue>.Builder issues)
{
if (actionObject is null)
{
return null;
}
var vendors = ImmutableArray<string>.Empty;
var justifications = ImmutableArray<string>.Empty;
if (actionObject.TryGetPropertyValue("vendors", out var vendorsNode))
{
vendors = NormalizeJsonStringArray(vendorsNode, $"{path}.vendors", issues);
}
if (actionObject.TryGetPropertyValue("justifications", out var justificationsNode))
{
justifications = NormalizeJsonStringArray(justificationsNode, $"{path}.justifications", issues);
}
return new PolicyRequireVexOptions(vendors, justifications);
}
private static ImmutableArray<string> NormalizeStringList(
List<string>? values,
string path,
ImmutableArray<PolicyIssue>.Builder issues)
{
if (values is null || values.Count == 0)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
foreach (var value in values)
{
var normalized = NormalizeOptionalString(value);
if (string.IsNullOrEmpty(normalized))
{
issues.Add(PolicyIssue.Warning("policy.list.blank", $"Blank entry detected; ignoring value at {path}.", path));
continue;
}
builder.Add(normalized);
}
return builder.ToImmutable()
.OrderBy(static item => item, StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
}
private static ImmutableArray<PolicySeverity> NormalizeSeverityList(
List<string>? values,
string path,
ImmutableArray<PolicyIssue>.Builder issues)
{
if (values is null || values.Count == 0)
{
return ImmutableArray<PolicySeverity>.Empty;
}
var builder = ImmutableArray.CreateBuilder<PolicySeverity>();
foreach (var value in values)
{
var normalized = NormalizeOptionalString(value);
if (string.IsNullOrEmpty(normalized))
{
issues.Add(PolicyIssue.Warning("policy.severity.blank", "Blank severity was ignored.", path));
continue;
}
if (SeverityMap.TryGetValue(normalized, out var severity))
{
builder.Add(severity);
}
else
{
issues.Add(PolicyIssue.Error("policy.severity.invalid", $"Unknown severity '{value}'.", path));
}
}
return builder.Distinct().OrderBy(static sev => sev).ToImmutableArray();
}
private static ImmutableArray<string> NormalizeJsonStringArray(
JsonNode? node,
string path,
ImmutableArray<PolicyIssue>.Builder issues)
{
if (node is null)
{
return ImmutableArray<string>.Empty;
}
if (node is JsonArray array)
{
var values = new List<string>(array.Count);
foreach (var element in array)
{
var text = ConvertNodeToString(element);
if (string.IsNullOrWhiteSpace(text))
{
issues.Add(PolicyIssue.Warning("policy.list.blank", $"Blank entry detected; ignoring value at {path}.", path));
}
else
{
values.Add(text);
}
}
return values
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static entry => entry, StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
}
var single = ConvertNodeToString(node);
return ImmutableArray.Create(single);
}
private static double? ParseDouble(JsonNode? node, string path, ImmutableArray<PolicyIssue>.Builder issues)
{
if (node is null)
{
return null;
}
if (node is JsonValue value)
{
if (value.TryGetValue(out double numeric))
{
return numeric;
}
if (value.TryGetValue(out string? text) && double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out numeric))
{
return numeric;
}
}
issues.Add(PolicyIssue.Warning("policy.number.invalid", $"Value '{node.ToJsonString()}' is not a valid number.", path));
return null;
}
private static DateTimeOffset? NormalizeTemporal(JsonNode? node, string path, ImmutableArray<PolicyIssue>.Builder issues)
{
if (node is null)
{
return null;
}
if (node is JsonValue value)
{
if (value.TryGetValue(out DateTimeOffset dto))
{
return dto;
}
if (value.TryGetValue(out DateTime dt))
{
return new DateTimeOffset(DateTime.SpecifyKind(dt, DateTimeKind.Utc));
}
if (value.TryGetValue(out string? text))
{
if (DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
{
return parsed;
}
if (DateTime.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsedDate))
{
return new DateTimeOffset(parsedDate);
}
}
}
issues.Add(PolicyIssue.Warning("policy.date.invalid", $"Value '{node.ToJsonString()}' is not a valid ISO-8601 timestamp.", path));
return null;
}
private static string? NormalizeRequiredString(
string? value,
string path,
string fieldDescription,
ImmutableArray<PolicyIssue>.Builder issues)
{
var normalized = NormalizeOptionalString(value);
if (!string.IsNullOrEmpty(normalized))
{
return normalized;
}
issues.Add(PolicyIssue.Error(
"policy.required",
$"{fieldDescription} is required.",
path));
return null;
}
private static string? NormalizeOptionalString(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
return value.Trim();
}
private static string ConvertNodeToString(JsonNode? node)
{
if (node is null)
{
return string.Empty;
}
return node switch
{
JsonValue value when value.TryGetValue(out string? text) => text ?? string.Empty,
JsonValue value when value.TryGetValue(out bool boolean) => boolean ? "true" : "false",
JsonValue value when value.TryGetValue(out double numeric) => numeric.ToString(CultureInfo.InvariantCulture),
JsonObject obj => obj.ToJsonString(),
JsonArray array => array.ToJsonString(),
_ => node.ToJsonString(),
};
}
private static ImmutableArray<PolicyIssue> SortIssues(ImmutableArray<PolicyIssue>.Builder issues)
{
return issues.ToImmutable()
.OrderBy(static issue => issue.Severity switch
{
PolicyIssueSeverity.Error => 0,
PolicyIssueSeverity.Warning => 1,
_ => 2,
})
.ThenBy(static issue => issue.Path, StringComparer.Ordinal)
.ThenBy(static issue => issue.Code, StringComparer.Ordinal)
.ToImmutableArray();
}
}
}

View File

@@ -0,0 +1,77 @@
using System;
using System.Collections.Immutable;
using System.Linq;
namespace StellaOps.Policy;
public sealed record PolicyDiagnosticsReport(
string Version,
int RuleCount,
int ErrorCount,
int WarningCount,
DateTimeOffset GeneratedAt,
ImmutableArray<PolicyIssue> Issues,
ImmutableArray<string> Recommendations);
public static class PolicyDiagnostics
{
public static PolicyDiagnosticsReport Create(PolicyBindingResult bindingResult, TimeProvider? timeProvider = null)
{
if (bindingResult is null)
{
throw new ArgumentNullException(nameof(bindingResult));
}
var time = (timeProvider ?? TimeProvider.System).GetUtcNow();
var errorCount = bindingResult.Issues.Count(static issue => issue.Severity == PolicyIssueSeverity.Error);
var warningCount = bindingResult.Issues.Count(static issue => issue.Severity == PolicyIssueSeverity.Warning);
var recommendations = BuildRecommendations(bindingResult.Document, errorCount, warningCount);
return new PolicyDiagnosticsReport(
bindingResult.Document.Version,
bindingResult.Document.Rules.Length,
errorCount,
warningCount,
time,
bindingResult.Issues,
recommendations);
}
private static ImmutableArray<string> BuildRecommendations(PolicyDocument document, int errorCount, int warningCount)
{
var messages = ImmutableArray.CreateBuilder<string>();
if (errorCount > 0)
{
messages.Add("Resolve policy errors before promoting the revision; fallback rules may be applied while errors remain.");
}
if (warningCount > 0)
{
messages.Add("Review policy warnings and ensure intentional overrides are documented.");
}
if (document.Rules.Length == 0)
{
messages.Add("Add at least one policy rule to enforce gating logic.");
}
var quietRules = document.Rules
.Where(static rule => rule.Action.Quiet)
.Select(static rule => rule.Name)
.ToArray();
if (quietRules.Length > 0)
{
messages.Add($"Quiet rules detected ({string.Join(", ", quietRules)}); verify scoring behaviour aligns with expectations.");
}
if (messages.Count == 0)
{
messages.Add("Policy validated successfully; no additional action required.");
}
return messages.ToImmutable();
}
}

View File

@@ -0,0 +1,211 @@
using System;
using System.Buffers;
using System.Collections.Immutable;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
namespace StellaOps.Policy;
public static class PolicyDigest
{
public static string Compute(PolicyDocument document)
{
if (document is null)
{
throw new ArgumentNullException(nameof(document));
}
var buffer = new ArrayBufferWriter<byte>();
using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions
{
SkipValidation = true,
}))
{
WriteDocument(writer, document);
}
var hash = SHA256.HashData(buffer.WrittenSpan);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static void WriteDocument(Utf8JsonWriter writer, PolicyDocument document)
{
writer.WriteStartObject();
writer.WriteString("version", document.Version);
if (!document.Metadata.IsEmpty)
{
writer.WritePropertyName("metadata");
writer.WriteStartObject();
foreach (var pair in document.Metadata.OrderBy(static kvp => kvp.Key, StringComparer.Ordinal))
{
writer.WriteString(pair.Key, pair.Value);
}
writer.WriteEndObject();
}
writer.WritePropertyName("rules");
writer.WriteStartArray();
foreach (var rule in document.Rules)
{
WriteRule(writer, rule);
}
writer.WriteEndArray();
writer.WriteEndObject();
writer.Flush();
}
private static void WriteRule(Utf8JsonWriter writer, PolicyRule rule)
{
writer.WriteStartObject();
writer.WriteString("name", rule.Name);
if (!string.IsNullOrWhiteSpace(rule.Identifier))
{
writer.WriteString("id", rule.Identifier);
}
if (!string.IsNullOrWhiteSpace(rule.Description))
{
writer.WriteString("description", rule.Description);
}
WriteMetadata(writer, rule.Metadata);
WriteSeverities(writer, rule.Severities);
WriteStringArray(writer, "environments", rule.Environments);
WriteStringArray(writer, "sources", rule.Sources);
WriteStringArray(writer, "vendors", rule.Vendors);
WriteStringArray(writer, "licenses", rule.Licenses);
WriteStringArray(writer, "tags", rule.Tags);
if (!rule.Match.IsEmpty)
{
writer.WritePropertyName("match");
writer.WriteStartObject();
WriteStringArray(writer, "images", rule.Match.Images);
WriteStringArray(writer, "repositories", rule.Match.Repositories);
WriteStringArray(writer, "packages", rule.Match.Packages);
WriteStringArray(writer, "purls", rule.Match.Purls);
WriteStringArray(writer, "cves", rule.Match.Cves);
WriteStringArray(writer, "paths", rule.Match.Paths);
WriteStringArray(writer, "layerDigests", rule.Match.LayerDigests);
WriteStringArray(writer, "usedByEntrypoint", rule.Match.UsedByEntrypoint);
writer.WriteEndObject();
}
WriteAction(writer, rule.Action);
if (rule.Expires is DateTimeOffset expires)
{
writer.WriteString("expires", expires.ToUniversalTime().ToString("O"));
}
if (!string.IsNullOrWhiteSpace(rule.Justification))
{
writer.WriteString("justification", rule.Justification);
}
writer.WriteEndObject();
}
private static void WriteAction(Utf8JsonWriter writer, PolicyAction action)
{
writer.WritePropertyName("action");
writer.WriteStartObject();
writer.WriteString("type", action.Type.ToString().ToLowerInvariant());
if (action.Quiet)
{
writer.WriteBoolean("quiet", true);
}
if (action.Ignore is { } ignore)
{
if (ignore.Until is DateTimeOffset until)
{
writer.WriteString("until", until.ToUniversalTime().ToString("O"));
}
if (!string.IsNullOrWhiteSpace(ignore.Justification))
{
writer.WriteString("justification", ignore.Justification);
}
}
if (action.Escalate is { } escalate)
{
if (escalate.MinimumSeverity is { } severity)
{
writer.WriteString("severity", severity.ToString());
}
if (escalate.RequireKev)
{
writer.WriteBoolean("kev", true);
}
if (escalate.MinimumEpss is double epss)
{
writer.WriteNumber("epss", epss);
}
}
if (action.RequireVex is { } requireVex)
{
WriteStringArray(writer, "vendors", requireVex.Vendors);
WriteStringArray(writer, "justifications", requireVex.Justifications);
}
writer.WriteEndObject();
}
private static void WriteMetadata(Utf8JsonWriter writer, ImmutableDictionary<string, string> metadata)
{
if (metadata.IsEmpty)
{
return;
}
writer.WritePropertyName("metadata");
writer.WriteStartObject();
foreach (var pair in metadata.OrderBy(static kvp => kvp.Key, StringComparer.Ordinal))
{
writer.WriteString(pair.Key, pair.Value);
}
writer.WriteEndObject();
}
private static void WriteSeverities(Utf8JsonWriter writer, ImmutableArray<PolicySeverity> severities)
{
if (severities.IsDefaultOrEmpty)
{
return;
}
writer.WritePropertyName("severity");
writer.WriteStartArray();
foreach (var severity in severities)
{
writer.WriteStringValue(severity.ToString());
}
writer.WriteEndArray();
}
private static void WriteStringArray(Utf8JsonWriter writer, string propertyName, ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
{
return;
}
writer.WritePropertyName(propertyName);
writer.WriteStartArray();
foreach (var value in values)
{
writer.WriteStringValue(value);
}
writer.WriteEndArray();
}
}

View File

@@ -0,0 +1,192 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Policy;
/// <summary>
/// Canonical representation of a StellaOps policy document.
/// </summary>
public sealed record PolicyDocument(
string Version,
ImmutableArray<PolicyRule> Rules,
ImmutableDictionary<string, string> Metadata)
{
public static PolicyDocument Empty { get; } = new(
PolicySchema.CurrentVersion,
ImmutableArray<PolicyRule>.Empty,
ImmutableDictionary<string, string>.Empty);
}
public static class PolicySchema
{
public const string SchemaId = "https://schemas.stella-ops.org/policy/policy-schema@1.json";
public const string CurrentVersion = "1.0";
public static PolicyDocumentFormat DetectFormat(string fileName)
{
if (fileName is null)
{
throw new ArgumentNullException(nameof(fileName));
}
var lower = fileName.Trim().ToLowerInvariant();
if (lower.EndsWith(".yaml", StringComparison.Ordinal) || lower.EndsWith(".yml", StringComparison.Ordinal))
{
return PolicyDocumentFormat.Yaml;
}
return PolicyDocumentFormat.Json;
}
}
public sealed record PolicyRule(
string Name,
string? Identifier,
string? Description,
PolicyAction Action,
ImmutableArray<PolicySeverity> Severities,
ImmutableArray<string> Environments,
ImmutableArray<string> Sources,
ImmutableArray<string> Vendors,
ImmutableArray<string> Licenses,
ImmutableArray<string> Tags,
PolicyRuleMatchCriteria Match,
DateTimeOffset? Expires,
string? Justification,
ImmutableDictionary<string, string> Metadata)
{
public static PolicyRuleMatchCriteria EmptyMatch { get; } = new(
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty);
public static PolicyRule Create(
string name,
PolicyAction action,
ImmutableArray<PolicySeverity> severities,
ImmutableArray<string> environments,
ImmutableArray<string> sources,
ImmutableArray<string> vendors,
ImmutableArray<string> licenses,
ImmutableArray<string> tags,
PolicyRuleMatchCriteria match,
DateTimeOffset? expires,
string? justification,
string? identifier = null,
string? description = null,
ImmutableDictionary<string, string>? metadata = null)
{
metadata ??= ImmutableDictionary<string, string>.Empty;
return new PolicyRule(
name,
identifier,
description,
action,
severities,
environments,
sources,
vendors,
licenses,
tags,
match,
expires,
justification,
metadata);
}
public bool MatchesAnyEnvironment => Environments.IsDefaultOrEmpty;
}
public sealed record PolicyRuleMatchCriteria(
ImmutableArray<string> Images,
ImmutableArray<string> Repositories,
ImmutableArray<string> Packages,
ImmutableArray<string> Purls,
ImmutableArray<string> Cves,
ImmutableArray<string> Paths,
ImmutableArray<string> LayerDigests,
ImmutableArray<string> UsedByEntrypoint)
{
public static PolicyRuleMatchCriteria Create(
ImmutableArray<string> images,
ImmutableArray<string> repositories,
ImmutableArray<string> packages,
ImmutableArray<string> purls,
ImmutableArray<string> cves,
ImmutableArray<string> paths,
ImmutableArray<string> layerDigests,
ImmutableArray<string> usedByEntrypoint)
=> new(
images,
repositories,
packages,
purls,
cves,
paths,
layerDigests,
usedByEntrypoint);
public static PolicyRuleMatchCriteria Empty { get; } = new(
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty,
ImmutableArray<string>.Empty);
public bool IsEmpty =>
Images.IsDefaultOrEmpty &&
Repositories.IsDefaultOrEmpty &&
Packages.IsDefaultOrEmpty &&
Purls.IsDefaultOrEmpty &&
Cves.IsDefaultOrEmpty &&
Paths.IsDefaultOrEmpty &&
LayerDigests.IsDefaultOrEmpty &&
UsedByEntrypoint.IsDefaultOrEmpty;
}
public sealed record PolicyAction(
PolicyActionType Type,
PolicyIgnoreOptions? Ignore,
PolicyEscalateOptions? Escalate,
PolicyRequireVexOptions? RequireVex,
bool Quiet);
public enum PolicyActionType
{
Block,
Ignore,
Warn,
Defer,
Escalate,
RequireVex,
}
public sealed record PolicyIgnoreOptions(DateTimeOffset? Until, string? Justification);
public sealed record PolicyEscalateOptions(
PolicySeverity? MinimumSeverity,
bool RequireKev,
double? MinimumEpss);
public sealed record PolicyRequireVexOptions(
ImmutableArray<string> Vendors,
ImmutableArray<string> Justifications);
public enum PolicySeverity
{
Critical,
High,
Medium,
Low,
Informational,
None,
Unknown,
}

View File

@@ -0,0 +1,270 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Policy;
public static class PolicyEvaluation
{
public static PolicyVerdict EvaluateFinding(PolicyDocument document, PolicyScoringConfig scoringConfig, PolicyFinding finding)
{
if (document is null)
{
throw new ArgumentNullException(nameof(document));
}
if (scoringConfig is null)
{
throw new ArgumentNullException(nameof(scoringConfig));
}
if (finding is null)
{
throw new ArgumentNullException(nameof(finding));
}
var severityWeight = scoringConfig.SeverityWeights.TryGetValue(finding.Severity, out var weight)
? weight
: scoringConfig.SeverityWeights.GetValueOrDefault(PolicySeverity.Unknown, 0);
foreach (var rule in document.Rules)
{
if (!RuleMatches(rule, finding))
{
continue;
}
return BuildVerdict(rule, finding, scoringConfig, severityWeight);
}
return PolicyVerdict.CreateBaseline(finding.FindingId, scoringConfig);
}
private static PolicyVerdict BuildVerdict(
PolicyRule rule,
PolicyFinding finding,
PolicyScoringConfig config,
double severityWeight)
{
var action = rule.Action;
var status = MapAction(action);
var notes = BuildNotes(action);
var inputs = ImmutableDictionary.CreateBuilder<string, double>(StringComparer.OrdinalIgnoreCase);
inputs["severityWeight"] = severityWeight;
double score = severityWeight;
string? quietedBy = null;
var quiet = false;
switch (status)
{
case PolicyVerdictStatus.Ignored:
score = Math.Max(0, severityWeight - config.IgnorePenalty);
inputs["ignorePenalty"] = config.IgnorePenalty;
break;
case PolicyVerdictStatus.Warned:
score = Math.Max(0, severityWeight - config.WarnPenalty);
inputs["warnPenalty"] = config.WarnPenalty;
break;
case PolicyVerdictStatus.Deferred:
score = Math.Max(0, severityWeight - (config.WarnPenalty / 2));
inputs["deferPenalty"] = config.WarnPenalty / 2;
break;
}
if (action.Quiet)
{
var quietAllowed = action.RequireVex is not null || action.Type == PolicyActionType.RequireVex;
if (quietAllowed)
{
score = Math.Max(0, score - config.QuietPenalty);
inputs["quietPenalty"] = config.QuietPenalty;
quietedBy = rule.Name;
quiet = true;
}
else
{
inputs.Remove("ignorePenalty");
var warnScore = Math.Max(0, severityWeight - config.WarnPenalty);
inputs["warnPenalty"] = config.WarnPenalty;
var warnNotes = AppendNote(notes, "Quiet flag ignored: rule must specify requireVex justifications.");
return new PolicyVerdict(
finding.FindingId,
PolicyVerdictStatus.Warned,
rule.Name,
action.Type.ToString(),
warnNotes,
warnScore,
config.Version,
inputs.ToImmutable(),
QuietedBy: null,
Quiet: false);
}
}
return new PolicyVerdict(
finding.FindingId,
status,
rule.Name,
action.Type.ToString(),
notes,
score,
config.Version,
inputs.ToImmutable(),
quietedBy,
quiet);
}
private static bool RuleMatches(PolicyRule rule, PolicyFinding finding)
{
if (!rule.Severities.IsDefaultOrEmpty && !rule.Severities.Contains(finding.Severity))
{
return false;
}
if (!Matches(rule.Environments, finding.Environment))
{
return false;
}
if (!Matches(rule.Sources, finding.Source))
{
return false;
}
if (!Matches(rule.Vendors, finding.Vendor))
{
return false;
}
if (!Matches(rule.Licenses, finding.License))
{
return false;
}
if (!RuleMatchCriteria(rule.Match, finding))
{
return false;
}
return true;
}
private static bool Matches(ImmutableArray<string> ruleValues, string? candidate)
{
if (ruleValues.IsDefaultOrEmpty)
{
return true;
}
if (string.IsNullOrWhiteSpace(candidate))
{
return false;
}
return ruleValues.Contains(candidate, StringComparer.OrdinalIgnoreCase);
}
private static bool RuleMatchCriteria(PolicyRuleMatchCriteria criteria, PolicyFinding finding)
{
if (!criteria.Images.IsDefaultOrEmpty && !ContainsValue(criteria.Images, finding.Image, StringComparer.OrdinalIgnoreCase))
{
return false;
}
if (!criteria.Repositories.IsDefaultOrEmpty && !ContainsValue(criteria.Repositories, finding.Repository, StringComparer.OrdinalIgnoreCase))
{
return false;
}
if (!criteria.Packages.IsDefaultOrEmpty && !ContainsValue(criteria.Packages, finding.Package, StringComparer.OrdinalIgnoreCase))
{
return false;
}
if (!criteria.Purls.IsDefaultOrEmpty && !ContainsValue(criteria.Purls, finding.Purl, StringComparer.OrdinalIgnoreCase))
{
return false;
}
if (!criteria.Cves.IsDefaultOrEmpty && !ContainsValue(criteria.Cves, finding.Cve, StringComparer.OrdinalIgnoreCase))
{
return false;
}
if (!criteria.Paths.IsDefaultOrEmpty && !ContainsValue(criteria.Paths, finding.Path, StringComparer.Ordinal))
{
return false;
}
if (!criteria.LayerDigests.IsDefaultOrEmpty && !ContainsValue(criteria.LayerDigests, finding.LayerDigest, StringComparer.OrdinalIgnoreCase))
{
return false;
}
if (!criteria.UsedByEntrypoint.IsDefaultOrEmpty)
{
var match = false;
foreach (var tag in criteria.UsedByEntrypoint)
{
if (finding.Tags.Contains(tag, StringComparer.OrdinalIgnoreCase))
{
match = true;
break;
}
}
if (!match)
{
return false;
}
}
return true;
}
private static bool ContainsValue(ImmutableArray<string> values, string? candidate, StringComparer comparer)
{
if (values.IsDefaultOrEmpty)
{
return true;
}
if (string.IsNullOrWhiteSpace(candidate))
{
return false;
}
return values.Contains(candidate, comparer);
}
private static PolicyVerdictStatus MapAction(PolicyAction action)
=> action.Type switch
{
PolicyActionType.Block => PolicyVerdictStatus.Blocked,
PolicyActionType.Ignore => PolicyVerdictStatus.Ignored,
PolicyActionType.Warn => PolicyVerdictStatus.Warned,
PolicyActionType.Defer => PolicyVerdictStatus.Deferred,
PolicyActionType.Escalate => PolicyVerdictStatus.Escalated,
PolicyActionType.RequireVex => PolicyVerdictStatus.RequiresVex,
_ => PolicyVerdictStatus.Pass,
};
private static string? BuildNotes(PolicyAction action)
{
if (action.Ignore is { } ignore && !string.IsNullOrWhiteSpace(ignore.Justification))
{
return ignore.Justification;
}
if (action.Escalate is { } escalate && escalate.MinimumSeverity is { } severity)
{
return $"Escalate >= {severity}";
}
return null;
}
private static string? AppendNote(string? existing, string addition)
=> string.IsNullOrWhiteSpace(existing) ? addition : string.Concat(existing, " | ", addition);
}

View File

@@ -0,0 +1,51 @@
using System.Collections.Immutable;
namespace StellaOps.Policy;
public sealed record PolicyFinding(
string FindingId,
PolicySeverity Severity,
string? Environment,
string? Source,
string? Vendor,
string? License,
string? Image,
string? Repository,
string? Package,
string? Purl,
string? Cve,
string? Path,
string? LayerDigest,
ImmutableArray<string> Tags)
{
public static PolicyFinding Create(
string findingId,
PolicySeverity severity,
string? environment = null,
string? source = null,
string? vendor = null,
string? license = null,
string? image = null,
string? repository = null,
string? package = null,
string? purl = null,
string? cve = null,
string? path = null,
string? layerDigest = null,
ImmutableArray<string>? tags = null)
=> new(
findingId,
severity,
environment,
source,
vendor,
license,
image,
repository,
package,
purl,
cve,
path,
layerDigest,
tags ?? ImmutableArray<string>.Empty);
}

View File

@@ -0,0 +1,28 @@
using System;
namespace StellaOps.Policy;
/// <summary>
/// Represents a validation or normalization issue discovered while processing a policy document.
/// </summary>
public sealed record PolicyIssue(string Code, string Message, PolicyIssueSeverity Severity, string Path)
{
public static PolicyIssue Error(string code, string message, string path)
=> new(code, message, PolicyIssueSeverity.Error, path);
public static PolicyIssue Warning(string code, string message, string path)
=> new(code, message, PolicyIssueSeverity.Warning, path);
public static PolicyIssue Info(string code, string message, string path)
=> new(code, message, PolicyIssueSeverity.Info, path);
public PolicyIssue EnsurePath(string fallbackPath)
=> string.IsNullOrWhiteSpace(Path) ? this with { Path = fallbackPath } : this;
}
public enum PolicyIssueSeverity
{
Error,
Warning,
Info,
}

View File

@@ -0,0 +1,18 @@
using System.Collections.Immutable;
namespace StellaOps.Policy;
public sealed record PolicyPreviewRequest(
string ImageDigest,
ImmutableArray<PolicyFinding> Findings,
ImmutableArray<PolicyVerdict> BaselineVerdicts,
PolicySnapshot? SnapshotOverride = null,
PolicySnapshotContent? ProposedPolicy = null);
public sealed record PolicyPreviewResponse(
bool Success,
string PolicyDigest,
string? RevisionId,
ImmutableArray<PolicyIssue> Issues,
ImmutableArray<PolicyVerdictDiff> Diffs,
int ChangedCount);

View File

@@ -0,0 +1,142 @@
using System;
using System.Collections.Generic;
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace StellaOps.Policy;
public sealed class PolicyPreviewService
{
private readonly PolicySnapshotStore _snapshotStore;
private readonly ILogger<PolicyPreviewService> _logger;
public PolicyPreviewService(PolicySnapshotStore snapshotStore, ILogger<PolicyPreviewService> logger)
{
_snapshotStore = snapshotStore ?? throw new ArgumentNullException(nameof(snapshotStore));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<PolicyPreviewResponse> PreviewAsync(PolicyPreviewRequest request, CancellationToken cancellationToken = default)
{
if (request is null)
{
throw new ArgumentNullException(nameof(request));
}
var (snapshot, bindingIssues) = await ResolveSnapshotAsync(request, cancellationToken).ConfigureAwait(false);
if (snapshot is null)
{
_logger.LogWarning("Policy preview failed: snapshot unavailable or validation errors. Issues={Count}", bindingIssues.Length);
return new PolicyPreviewResponse(false, string.Empty, null, bindingIssues, ImmutableArray<PolicyVerdictDiff>.Empty, 0);
}
var projected = Evaluate(snapshot.Document, snapshot.ScoringConfig, request.Findings);
var baseline = BuildBaseline(request.BaselineVerdicts, projected, snapshot.ScoringConfig);
var diffs = BuildDiffs(baseline, projected);
var changed = diffs.Count(static diff => diff.Changed);
_logger.LogDebug("Policy preview computed for {ImageDigest}. Changed={Changed}", request.ImageDigest, changed);
return new PolicyPreviewResponse(true, snapshot.Digest, snapshot.RevisionId, bindingIssues, diffs, changed);
}
private async Task<(PolicySnapshot? Snapshot, ImmutableArray<PolicyIssue> Issues)> ResolveSnapshotAsync(PolicyPreviewRequest request, CancellationToken cancellationToken)
{
if (request.ProposedPolicy is not null)
{
var binding = PolicyBinder.Bind(request.ProposedPolicy.Content, request.ProposedPolicy.Format);
if (!binding.Success)
{
return (null, binding.Issues);
}
var digest = PolicyDigest.Compute(binding.Document);
var snapshot = new PolicySnapshot(
request.SnapshotOverride?.RevisionNumber + 1 ?? 0,
request.SnapshotOverride?.RevisionId ?? "preview",
digest,
DateTimeOffset.UtcNow,
request.ProposedPolicy.Actor,
request.ProposedPolicy.Format,
binding.Document,
binding.Issues,
PolicyScoringConfig.Default);
return (snapshot, binding.Issues);
}
if (request.SnapshotOverride is not null)
{
return (request.SnapshotOverride, ImmutableArray<PolicyIssue>.Empty);
}
var latest = await _snapshotStore.GetLatestAsync(cancellationToken).ConfigureAwait(false);
if (latest is not null)
{
return (latest, ImmutableArray<PolicyIssue>.Empty);
}
return (null, ImmutableArray.Create(PolicyIssue.Error("policy.preview.snapshot_missing", "No policy snapshot is available for preview.", "$")));
}
private static ImmutableArray<PolicyVerdict> Evaluate(PolicyDocument document, PolicyScoringConfig scoringConfig, ImmutableArray<PolicyFinding> findings)
{
if (findings.IsDefaultOrEmpty)
{
return ImmutableArray<PolicyVerdict>.Empty;
}
var results = ImmutableArray.CreateBuilder<PolicyVerdict>(findings.Length);
foreach (var finding in findings)
{
var verdict = PolicyEvaluation.EvaluateFinding(document, scoringConfig, finding);
results.Add(verdict);
}
return results.ToImmutable();
}
private static ImmutableDictionary<string, PolicyVerdict> BuildBaseline(ImmutableArray<PolicyVerdict> baseline, ImmutableArray<PolicyVerdict> projected, PolicyScoringConfig scoringConfig)
{
var builder = ImmutableDictionary.CreateBuilder<string, PolicyVerdict>(StringComparer.Ordinal);
if (!baseline.IsDefaultOrEmpty)
{
foreach (var verdict in baseline)
{
if (!string.IsNullOrEmpty(verdict.FindingId) && !builder.ContainsKey(verdict.FindingId))
{
builder.Add(verdict.FindingId, verdict);
}
}
}
foreach (var verdict in projected)
{
if (!builder.ContainsKey(verdict.FindingId))
{
builder.Add(verdict.FindingId, PolicyVerdict.CreateBaseline(verdict.FindingId, scoringConfig));
}
}
return builder.ToImmutable();
}
private static ImmutableArray<PolicyVerdictDiff> BuildDiffs(ImmutableDictionary<string, PolicyVerdict> baseline, ImmutableArray<PolicyVerdict> projected)
{
var diffs = ImmutableArray.CreateBuilder<PolicyVerdictDiff>(projected.Length);
foreach (var verdict in projected.OrderBy(static v => v.FindingId, StringComparer.Ordinal))
{
var baseVerdict = baseline.TryGetValue(verdict.FindingId, out var existing)
? existing
: new PolicyVerdict(verdict.FindingId, PolicyVerdictStatus.Pass);
diffs.Add(new PolicyVerdictDiff(baseVerdict, verdict));
}
return diffs.ToImmutable();
}
}

View File

@@ -0,0 +1,30 @@
using System;
using System.IO;
using System.Reflection;
using System.Text;
namespace StellaOps.Policy;
public static class PolicySchemaResource
{
private const string SchemaResourceName = "StellaOps.Policy.Schemas.policy-schema@1.json";
public static Stream OpenSchemaStream()
{
var assembly = Assembly.GetExecutingAssembly();
var stream = assembly.GetManifestResourceStream(SchemaResourceName);
if (stream is null)
{
throw new InvalidOperationException($"Unable to locate embedded schema resource '{SchemaResourceName}'.");
}
return stream;
}
public static string ReadSchemaJson()
{
using var stream = OpenSchemaStream();
using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
return reader.ReadToEnd();
}
}

View File

@@ -0,0 +1,16 @@
using System.Collections.Immutable;
namespace StellaOps.Policy;
public sealed record PolicyScoringConfig(
string Version,
ImmutableDictionary<PolicySeverity, double> SeverityWeights,
double QuietPenalty,
double WarnPenalty,
double IgnorePenalty,
ImmutableDictionary<string, double> TrustOverrides)
{
public static string BaselineVersion => "1.0";
public static PolicyScoringConfig Default { get; } = PolicyScoringConfigBinder.LoadDefault();
}

View File

@@ -0,0 +1,266 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace StellaOps.Policy;
public sealed record PolicyScoringBindingResult(
bool Success,
PolicyScoringConfig? Config,
ImmutableArray<PolicyIssue> Issues);
public static class PolicyScoringConfigBinder
{
private const string DefaultResourceName = "StellaOps.Policy.Schemas.policy-scoring-default.json";
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true,
};
private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.IgnoreUnmatchedProperties()
.Build();
public static PolicyScoringConfig LoadDefault()
{
var assembly = Assembly.GetExecutingAssembly();
using var stream = assembly.GetManifestResourceStream(DefaultResourceName)
?? throw new InvalidOperationException($"Embedded resource '{DefaultResourceName}' not found.");
using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
var json = reader.ReadToEnd();
var binding = Bind(json, PolicyDocumentFormat.Json);
if (!binding.Success || binding.Config is null)
{
throw new InvalidOperationException("Failed to load default policy scoring configuration.");
}
return binding.Config;
}
public static PolicyScoringBindingResult Bind(string content, PolicyDocumentFormat format)
{
if (string.IsNullOrWhiteSpace(content))
{
var issue = PolicyIssue.Error("scoring.empty", "Scoring configuration content is empty.", "$");
return new PolicyScoringBindingResult(false, null, ImmutableArray.Create(issue));
}
try
{
var root = Parse(content, format);
if (root is not JsonObject obj)
{
var issue = PolicyIssue.Error("scoring.invalid", "Scoring configuration must be a JSON object.", "$");
return new PolicyScoringBindingResult(false, null, ImmutableArray.Create(issue));
}
var issues = ImmutableArray.CreateBuilder<PolicyIssue>();
var config = BuildConfig(obj, issues);
var hasErrors = issues.Any(issue => issue.Severity == PolicyIssueSeverity.Error);
return new PolicyScoringBindingResult(!hasErrors, config, issues.ToImmutable());
}
catch (JsonException ex)
{
var issue = PolicyIssue.Error("scoring.parse.json", $"Failed to parse scoring JSON: {ex.Message}", "$");
return new PolicyScoringBindingResult(false, null, ImmutableArray.Create(issue));
}
catch (YamlDotNet.Core.YamlException ex)
{
var issue = PolicyIssue.Error("scoring.parse.yaml", $"Failed to parse scoring YAML: {ex.Message}", "$");
return new PolicyScoringBindingResult(false, null, ImmutableArray.Create(issue));
}
}
private static JsonNode? Parse(string content, PolicyDocumentFormat format)
{
return format switch
{
PolicyDocumentFormat.Json => JsonNode.Parse(content, new JsonNodeOptions { PropertyNameCaseInsensitive = true }),
PolicyDocumentFormat.Yaml => ConvertYamlToJsonNode(content),
_ => throw new ArgumentOutOfRangeException(nameof(format), format, "Unsupported scoring configuration format."),
};
}
private static JsonNode? ConvertYamlToJsonNode(string content)
{
var yamlObject = YamlDeserializer.Deserialize<object?>(content);
return PolicyBinderUtilities.ConvertYamlObject(yamlObject);
}
private static PolicyScoringConfig BuildConfig(JsonObject obj, ImmutableArray<PolicyIssue>.Builder issues)
{
var version = ReadString(obj, "version", issues, required: true) ?? PolicyScoringConfig.BaselineVersion;
var severityWeights = ReadSeverityWeights(obj, issues);
var quietPenalty = ReadDouble(obj, "quietPenalty", issues, defaultValue: 45);
var warnPenalty = ReadDouble(obj, "warnPenalty", issues, defaultValue: 15);
var ignorePenalty = ReadDouble(obj, "ignorePenalty", issues, defaultValue: 35);
var trustOverrides = ReadTrustOverrides(obj, issues);
return new PolicyScoringConfig(
version,
severityWeights,
quietPenalty,
warnPenalty,
ignorePenalty,
trustOverrides);
}
private static ImmutableDictionary<PolicySeverity, double> ReadSeverityWeights(JsonObject obj, ImmutableArray<PolicyIssue>.Builder issues)
{
if (!obj.TryGetPropertyValue("severityWeights", out var node) || node is not JsonObject severityObj)
{
issues.Add(PolicyIssue.Error("scoring.severityWeights.missing", "severityWeights section is required.", "$.severityWeights"));
return ImmutableDictionary<PolicySeverity, double>.Empty;
}
var builder = ImmutableDictionary.CreateBuilder<PolicySeverity, double>();
foreach (var severity in Enum.GetValues<PolicySeverity>())
{
var key = severity.ToString();
if (!severityObj.TryGetPropertyValue(key, out var valueNode))
{
issues.Add(PolicyIssue.Warning("scoring.severityWeights.default", $"Severity '{key}' not specified; defaulting to 0.", $"$.severityWeights.{key}"));
builder[severity] = 0;
continue;
}
var value = ExtractDouble(valueNode, issues, $"$.severityWeights.{key}");
builder[severity] = value;
}
return builder.ToImmutable();
}
private static double ReadDouble(JsonObject obj, string property, ImmutableArray<PolicyIssue>.Builder issues, double defaultValue)
{
if (!obj.TryGetPropertyValue(property, out var node))
{
issues.Add(PolicyIssue.Warning("scoring.numeric.default", $"{property} not specified; defaulting to {defaultValue:0.##}.", $"$.{property}"));
return defaultValue;
}
return ExtractDouble(node, issues, $"$.{property}");
}
private static double ExtractDouble(JsonNode? node, ImmutableArray<PolicyIssue>.Builder issues, string path)
{
if (node is null)
{
issues.Add(PolicyIssue.Warning("scoring.numeric.null", $"Value at {path} missing; defaulting to 0.", path));
return 0;
}
if (node is JsonValue value)
{
if (value.TryGetValue(out double number))
{
return number;
}
if (value.TryGetValue(out string? text) && double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out number))
{
return number;
}
}
issues.Add(PolicyIssue.Error("scoring.numeric.invalid", $"Value at {path} is not numeric.", path));
return 0;
}
private static ImmutableDictionary<string, double> ReadTrustOverrides(JsonObject obj, ImmutableArray<PolicyIssue>.Builder issues)
{
if (!obj.TryGetPropertyValue("trustOverrides", out var node) || node is not JsonObject trustObj)
{
return ImmutableDictionary<string, double>.Empty;
}
var builder = ImmutableDictionary.CreateBuilder<string, double>(StringComparer.OrdinalIgnoreCase);
foreach (var pair in trustObj)
{
var value = ExtractDouble(pair.Value, issues, $"$.trustOverrides.{pair.Key}");
builder[pair.Key] = value;
}
return builder.ToImmutable();
}
private static string? ReadString(JsonObject obj, string property, ImmutableArray<PolicyIssue>.Builder issues, bool required)
{
if (!obj.TryGetPropertyValue(property, out var node) || node is null)
{
if (required)
{
issues.Add(PolicyIssue.Error("scoring.string.missing", $"{property} is required.", $"$.{property}"));
}
return null;
}
if (node is JsonValue value && value.TryGetValue(out string? text))
{
return text?.Trim();
}
issues.Add(PolicyIssue.Error("scoring.string.invalid", $"{property} must be a string.", $"$.{property}"));
return null;
}
}
internal static class PolicyBinderUtilities
{
public static JsonNode? ConvertYamlObject(object? value)
{
switch (value)
{
case null:
return null;
case string s:
return JsonValue.Create(s);
case bool b:
return JsonValue.Create(b);
case sbyte or byte or short or ushort or int or uint or long or ulong or float or double or decimal:
return JsonValue.Create(Convert.ToDouble(value, CultureInfo.InvariantCulture));
case IDictionary dictionary:
{
var obj = new JsonObject();
foreach (DictionaryEntry entry in dictionary)
{
if (entry.Key is null)
{
continue;
}
obj[entry.Key.ToString()!] = ConvertYamlObject(entry.Value);
}
return obj;
}
case IEnumerable enumerable:
{
var array = new JsonArray();
foreach (var item in enumerable)
{
array.Add(ConvertYamlObject(item));
}
return array;
}
default:
return JsonValue.Create(value.ToString());
}
}
}

View File

@@ -0,0 +1,29 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Policy;
public sealed record PolicySnapshot(
long RevisionNumber,
string RevisionId,
string Digest,
DateTimeOffset CreatedAt,
string? CreatedBy,
PolicyDocumentFormat Format,
PolicyDocument Document,
ImmutableArray<PolicyIssue> Issues,
PolicyScoringConfig ScoringConfig);
public sealed record PolicySnapshotContent(
string Content,
PolicyDocumentFormat Format,
string? Actor,
string? Source,
string? Description);
public sealed record PolicySnapshotSaveResult(
bool Success,
bool Created,
string Digest,
PolicySnapshot? Snapshot,
PolicyBindingResult BindingResult);

View File

@@ -0,0 +1,101 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace StellaOps.Policy;
public sealed class PolicySnapshotStore
{
private readonly IPolicySnapshotRepository _snapshotRepository;
private readonly IPolicyAuditRepository _auditRepository;
private readonly TimeProvider _timeProvider;
private readonly ILogger<PolicySnapshotStore> _logger;
private readonly SemaphoreSlim _mutex = new(1, 1);
public PolicySnapshotStore(
IPolicySnapshotRepository snapshotRepository,
IPolicyAuditRepository auditRepository,
TimeProvider? timeProvider,
ILogger<PolicySnapshotStore> logger)
{
_snapshotRepository = snapshotRepository ?? throw new ArgumentNullException(nameof(snapshotRepository));
_auditRepository = auditRepository ?? throw new ArgumentNullException(nameof(auditRepository));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<PolicySnapshotSaveResult> SaveAsync(PolicySnapshotContent content, CancellationToken cancellationToken = default)
{
if (content is null)
{
throw new ArgumentNullException(nameof(content));
}
var bindingResult = PolicyBinder.Bind(content.Content, content.Format);
if (!bindingResult.Success)
{
_logger.LogWarning("Policy snapshot rejected due to validation errors (Format: {Format})", content.Format);
return new PolicySnapshotSaveResult(false, false, string.Empty, null, bindingResult);
}
var digest = PolicyDigest.Compute(bindingResult.Document);
await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var latest = await _snapshotRepository.GetLatestAsync(cancellationToken).ConfigureAwait(false);
if (latest is not null && string.Equals(latest.Digest, digest, StringComparison.Ordinal))
{
_logger.LogInformation("Policy snapshot unchanged; digest {Digest} matches revision {RevisionId}", digest, latest.RevisionId);
return new PolicySnapshotSaveResult(true, false, digest, latest, bindingResult);
}
var revisionNumber = (latest?.RevisionNumber ?? 0) + 1;
var revisionId = $"rev-{revisionNumber}";
var createdAt = _timeProvider.GetUtcNow();
var scoringConfig = PolicyScoringConfig.Default;
var snapshot = new PolicySnapshot(
revisionNumber,
revisionId,
digest,
createdAt,
content.Actor,
content.Format,
bindingResult.Document,
bindingResult.Issues,
scoringConfig);
await _snapshotRepository.AddAsync(snapshot, cancellationToken).ConfigureAwait(false);
var auditMessage = content.Description ?? "Policy snapshot created";
var auditEntry = new PolicyAuditEntry(
Guid.NewGuid(),
createdAt,
"snapshot.created",
revisionId,
digest,
content.Actor,
auditMessage);
await _auditRepository.AddAsync(auditEntry, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Policy snapshot saved. Revision {RevisionId}, digest {Digest}, issues {IssueCount}",
revisionId,
digest,
bindingResult.Issues.Length);
return new PolicySnapshotSaveResult(true, true, digest, snapshot, bindingResult);
}
finally
{
_mutex.Release();
}
}
public Task<PolicySnapshot?> GetLatestAsync(CancellationToken cancellationToken = default)
=> _snapshotRepository.GetLatestAsync(cancellationToken);
}

View File

@@ -0,0 +1,241 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Policy;
public sealed record PolicyValidationCliOptions
{
public IReadOnlyList<string> Inputs { get; init; } = Array.Empty<string>();
/// <summary>
/// Writes machine-readable JSON instead of human-formatted text.
/// </summary>
public bool OutputJson { get; init; }
/// <summary>
/// When enabled, warnings cause a non-zero exit code.
/// </summary>
public bool Strict { get; init; }
}
public sealed record PolicyValidationFileResult(
string Path,
PolicyBindingResult BindingResult,
PolicyDiagnosticsReport Diagnostics);
public sealed class PolicyValidationCli
{
private readonly TextWriter _output;
private readonly TextWriter _error;
public PolicyValidationCli(TextWriter? output = null, TextWriter? error = null)
{
_output = output ?? Console.Out;
_error = error ?? Console.Error;
}
public async Task<int> RunAsync(PolicyValidationCliOptions options, CancellationToken cancellationToken = default)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (options.Inputs.Count == 0)
{
await _error.WriteLineAsync("No input files provided. Supply one or more policy file paths.");
return 64; // EX_USAGE
}
var results = new List<PolicyValidationFileResult>();
foreach (var input in options.Inputs)
{
cancellationToken.ThrowIfCancellationRequested();
var resolvedPaths = ResolveInput(input);
if (resolvedPaths.Count == 0)
{
await _error.WriteLineAsync($"No files matched '{input}'.");
continue;
}
foreach (var path in resolvedPaths)
{
cancellationToken.ThrowIfCancellationRequested();
var format = PolicySchema.DetectFormat(path);
var content = await File.ReadAllTextAsync(path, cancellationToken);
var bindingResult = PolicyBinder.Bind(content, format);
var diagnostics = PolicyDiagnostics.Create(bindingResult);
results.Add(new PolicyValidationFileResult(path, bindingResult, diagnostics));
}
}
if (results.Count == 0)
{
await _error.WriteLineAsync("No files were processed.");
return 65; // EX_DATAERR
}
if (options.OutputJson)
{
WriteJson(results);
}
else
{
await WriteTextAsync(results, cancellationToken);
}
var hasErrors = results.Any(static result => !result.BindingResult.Success);
var hasWarnings = results.Any(static result => result.BindingResult.Issues.Any(static issue => issue.Severity == PolicyIssueSeverity.Warning));
if (hasErrors)
{
return 1;
}
if (options.Strict && hasWarnings)
{
return 2;
}
return 0;
}
private async Task WriteTextAsync(IReadOnlyList<PolicyValidationFileResult> results, CancellationToken cancellationToken)
{
foreach (var result in results)
{
cancellationToken.ThrowIfCancellationRequested();
var relativePath = MakeRelative(result.Path);
await _output.WriteLineAsync($"{relativePath} [{result.BindingResult.Format}]");
if (result.BindingResult.Issues.Length == 0)
{
await _output.WriteLineAsync(" OK");
continue;
}
foreach (var issue in result.BindingResult.Issues)
{
var severity = issue.Severity.ToString().ToUpperInvariant().PadRight(7);
await _output.WriteLineAsync($" {severity} {issue.Path} :: {issue.Message} ({issue.Code})");
}
}
}
private void WriteJson(IReadOnlyList<PolicyValidationFileResult> results)
{
var payload = results.Select(static result => new
{
path = result.Path,
format = result.BindingResult.Format.ToString().ToLowerInvariant(),
success = result.BindingResult.Success,
issues = result.BindingResult.Issues.Select(static issue => new
{
code = issue.Code,
message = issue.Message,
severity = issue.Severity.ToString().ToLowerInvariant(),
path = issue.Path,
}),
diagnostics = new
{
version = result.Diagnostics.Version,
ruleCount = result.Diagnostics.RuleCount,
errorCount = result.Diagnostics.ErrorCount,
warningCount = result.Diagnostics.WarningCount,
generatedAt = result.Diagnostics.GeneratedAt,
recommendations = result.Diagnostics.Recommendations,
},
})
.ToArray();
var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions
{
WriteIndented = true,
});
_output.WriteLine(json);
}
private static IReadOnlyList<string> ResolveInput(string input)
{
if (string.IsNullOrWhiteSpace(input))
{
return Array.Empty<string>();
}
var expanded = Environment.ExpandEnvironmentVariables(input.Trim());
if (File.Exists(expanded))
{
return new[] { Path.GetFullPath(expanded) };
}
if (Directory.Exists(expanded))
{
return Directory.EnumerateFiles(expanded, "*.*", SearchOption.TopDirectoryOnly)
.Where(static path => MatchesPolicyExtension(path))
.OrderBy(static path => path, StringComparer.OrdinalIgnoreCase)
.Select(Path.GetFullPath)
.ToArray();
}
var directory = Path.GetDirectoryName(expanded);
var searchPattern = Path.GetFileName(expanded);
if (string.IsNullOrEmpty(searchPattern))
{
return Array.Empty<string>();
}
if (string.IsNullOrEmpty(directory))
{
directory = ".";
}
if (!Directory.Exists(directory))
{
return Array.Empty<string>();
}
return Directory.EnumerateFiles(directory, searchPattern, SearchOption.TopDirectoryOnly)
.Where(static path => MatchesPolicyExtension(path))
.OrderBy(static path => path, StringComparer.OrdinalIgnoreCase)
.Select(Path.GetFullPath)
.ToArray();
}
private static bool MatchesPolicyExtension(string path)
{
var extension = Path.GetExtension(path);
return extension.Equals(".yaml", StringComparison.OrdinalIgnoreCase)
|| extension.Equals(".yml", StringComparison.OrdinalIgnoreCase)
|| extension.Equals(".json", StringComparison.OrdinalIgnoreCase);
}
private static string MakeRelative(string path)
{
try
{
var fullPath = Path.GetFullPath(path);
var current = Directory.GetCurrentDirectory();
if (fullPath.StartsWith(current, StringComparison.OrdinalIgnoreCase))
{
return fullPath[current.Length..].TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
return fullPath;
}
catch
{
return path;
}
}
}

View File

@@ -0,0 +1,80 @@
using System;
using System.Collections.Immutable;
namespace StellaOps.Policy;
public enum PolicyVerdictStatus
{
Pass,
Blocked,
Ignored,
Warned,
Deferred,
Escalated,
RequiresVex,
}
public sealed record PolicyVerdict(
string FindingId,
PolicyVerdictStatus Status,
string? RuleName = null,
string? RuleAction = null,
string? Notes = null,
double Score = 0,
string ConfigVersion = "1.0",
ImmutableDictionary<string, double>? Inputs = null,
string? QuietedBy = null,
bool Quiet = false)
{
public static PolicyVerdict CreateBaseline(string findingId, PolicyScoringConfig scoringConfig)
{
var inputs = ImmutableDictionary<string, double>.Empty;
return new PolicyVerdict(
findingId,
PolicyVerdictStatus.Pass,
RuleName: null,
RuleAction: null,
Notes: null,
Score: 0,
ConfigVersion: scoringConfig.Version,
Inputs: inputs,
QuietedBy: null,
Quiet: false);
}
public ImmutableDictionary<string, double> GetInputs()
=> Inputs ?? ImmutableDictionary<string, double>.Empty;
}
public sealed record PolicyVerdictDiff(
PolicyVerdict Baseline,
PolicyVerdict Projected)
{
public bool Changed
{
get
{
if (Baseline.Status != Projected.Status)
{
return true;
}
if (!string.Equals(Baseline.RuleName, Projected.RuleName, StringComparison.Ordinal))
{
return true;
}
if (Math.Abs(Baseline.Score - Projected.Score) > 0.0001)
{
return true;
}
if (!string.Equals(Baseline.QuietedBy, Projected.QuietedBy, StringComparison.Ordinal))
{
return true;
}
return false;
}
}
}

View File

@@ -0,0 +1,176 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://schemas.stella-ops.org/policy/policy-schema@1.json",
"title": "StellaOps Policy Schema v1",
"type": "object",
"required": ["version", "rules"],
"properties": {
"version": {
"type": ["string", "number"],
"enum": ["1", "1.0", 1, 1.0]
},
"description": {
"type": "string"
},
"metadata": {
"type": "object",
"additionalProperties": {
"type": ["string", "number", "boolean"]
}
},
"rules": {
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/$defs/rule"
}
}
},
"additionalProperties": true,
"$defs": {
"identifier": {
"type": "string",
"minLength": 1
},
"severity": {
"type": "string",
"enum": ["Critical", "High", "Medium", "Low", "Informational", "None", "Unknown"]
},
"stringArray": {
"type": "array",
"items": {
"type": "string",
"minLength": 1
},
"uniqueItems": true
},
"rule": {
"type": "object",
"required": ["name", "action"],
"properties": {
"id": {
"$ref": "#/$defs/identifier"
},
"name": {
"type": "string",
"minLength": 1
},
"description": {
"type": "string"
},
"severity": {
"type": "array",
"items": {
"$ref": "#/$defs/severity"
},
"uniqueItems": true
},
"sources": {
"$ref": "#/$defs/stringArray"
},
"vendors": {
"$ref": "#/$defs/stringArray"
},
"licenses": {
"$ref": "#/$defs/stringArray"
},
"tags": {
"$ref": "#/$defs/stringArray"
},
"environments": {
"$ref": "#/$defs/stringArray"
},
"images": {
"$ref": "#/$defs/stringArray"
},
"repositories": {
"$ref": "#/$defs/stringArray"
},
"packages": {
"$ref": "#/$defs/stringArray"
},
"purls": {
"$ref": "#/$defs/stringArray"
},
"cves": {
"$ref": "#/$defs/stringArray"
},
"paths": {
"$ref": "#/$defs/stringArray"
},
"layerDigests": {
"$ref": "#/$defs/stringArray"
},
"usedByEntrypoint": {
"$ref": "#/$defs/stringArray"
},
"justification": {
"type": "string"
},
"quiet": {
"type": "boolean"
},
"action": {
"oneOf": [
{
"type": "string",
"enum": ["block", "fail", "deny", "ignore", "warn", "defer", "escalate", "requireVex"]
},
{
"type": "object",
"required": ["type"],
"properties": {
"type": {
"type": "string"
},
"quiet": {
"type": "boolean"
},
"until": {
"type": "string",
"format": "date-time"
},
"justification": {
"type": "string"
},
"severity": {
"$ref": "#/$defs/severity"
},
"vendors": {
"$ref": "#/$defs/stringArray"
},
"justifications": {
"$ref": "#/$defs/stringArray"
},
"epss": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"kev": {
"type": "boolean"
}
},
"additionalProperties": true
}
]
},
"expires": {
"type": "string",
"format": "date-time"
},
"until": {
"type": "string",
"format": "date-time"
},
"metadata": {
"type": "object",
"additionalProperties": {
"type": ["string", "number", "boolean"]
}
}
},
"additionalProperties": true
}
}
}

View File

@@ -0,0 +1,21 @@
{
"version": "1.0",
"severityWeights": {
"Critical": 90.0,
"High": 75.0,
"Medium": 50.0,
"Low": 25.0,
"Informational": 10.0,
"None": 0.0,
"Unknown": 60.0
},
"quietPenalty": 45.0,
"warnPenalty": 15.0,
"ignorePenalty": 35.0,
"trustOverrides": {
"vendor": 1.0,
"distro": 0.85,
"platform": 0.75,
"community": 0.65
}
}

View File

@@ -3,5 +3,18 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.0-beta5.25306.1" />
<PackageReference Include="YamlDotNet" Version="13.7.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Schemas\policy-schema@1.json" />
<EmbeddedResource Include="Schemas\policy-scoring-default.json" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,14 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Policy;
public interface IPolicySnapshotRepository
{
Task<PolicySnapshot?> GetLatestAsync(CancellationToken cancellationToken = default);
Task<IReadOnlyList<PolicySnapshot>> ListAsync(int limit, CancellationToken cancellationToken = default);
Task AddAsync(PolicySnapshot snapshot, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,65 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Policy;
public sealed class InMemoryPolicySnapshotRepository : IPolicySnapshotRepository
{
private readonly List<PolicySnapshot> _snapshots = new();
private readonly SemaphoreSlim _mutex = new(1, 1);
public async Task AddAsync(PolicySnapshot snapshot, CancellationToken cancellationToken = default)
{
if (snapshot is null)
{
throw new ArgumentNullException(nameof(snapshot));
}
await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_snapshots.Add(snapshot);
_snapshots.Sort(static (left, right) => left.RevisionNumber.CompareTo(right.RevisionNumber));
}
finally
{
_mutex.Release();
}
}
public async Task<PolicySnapshot?> GetLatestAsync(CancellationToken cancellationToken = default)
{
await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
return _snapshots.Count == 0 ? null : _snapshots[^1];
}
finally
{
_mutex.Release();
}
}
public async Task<IReadOnlyList<PolicySnapshot>> ListAsync(int limit, CancellationToken cancellationToken = default)
{
await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
IEnumerable<PolicySnapshot> query = _snapshots;
if (limit > 0)
{
query = query.TakeLast(limit);
}
return query.ToImmutableArray();
}
finally
{
_mutex.Release();
}
}
}

View File

@@ -2,12 +2,17 @@
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| POLICY-CORE-09-001 | TODO | Policy Guild | SCANNER-WEB-09-101 | Define YAML schema/binder, diagnostics, CLI validation for policy files. | Schema doc published; binder loads sample policy; validation errors actionable. |
| POLICY-CORE-09-002 | TODO | Policy Guild | POLICY-CORE-09-001 | Implement policy snapshot store + revision digests + audit logging. | Snapshots persisted with digest; tests compare revisions; audit entries created. |
| POLICY-CORE-09-003 | TODO | Policy Guild | POLICY-CORE-09-002 | `/policy/preview` API (image digest → projected verdict delta). | Preview returns diff JSON; integration tests with mocked report; docs updated. |
| POLICY-CORE-09-001 | DONE | Policy Guild | SCANNER-WEB-09-101 | Define YAML schema/binder, diagnostics, CLI validation for policy files. | Schema doc published; binder loads sample policy; validation errors actionable. |
| POLICY-CORE-09-002 | DONE | Policy Guild | POLICY-CORE-09-001 | Implement policy snapshot store + revision digests + audit logging. | Snapshots persisted with digest; tests compare revisions; audit entries created. |
| POLICY-CORE-09-003 | DONE | Policy Guild | POLICY-CORE-09-002 | `/policy/preview` API (image digest → projected verdict delta). | Preview returns diff JSON; integration tests with mocked report; docs updated. |
| POLICY-CORE-09-004 | TODO | Policy Guild | POLICY-CORE-09-001 | Versioned scoring config with schema validation, trust table, and golden fixtures. | Scoring config documented; fixtures stored; validation CLI passes. |
| POLICY-CORE-09-005 | TODO | Policy Guild | POLICY-CORE-09-004 | Scoring/quiet engine compute score, enforce VEX-only quiet rules, emit inputs and provenance. | Engine unit tests cover severity weighting; outputs include provenance data. |
| POLICY-CORE-09-006 | TODO | Policy Guild | POLICY-CORE-09-005 | Unknown state & confidence decay deterministic bands surfaced in policy outputs. | Confidence decay tests pass; docs updated; preview endpoint displays banding. |
| POLICY-CORE-09-004 | TODO | Policy Guild | POLICY-CORE-09-001 | Versioned scoring config (weights, trust table, reachability buckets) with schema validation, binder, and golden fixtures. | Config serialized with semantic version, binder loads defaults, fixtures assert deterministic hash. |
| POLICY-CORE-09-005 | TODO | Policy Guild | POLICY-CORE-09-004, POLICY-CORE-09-002 | Implement scoring/quiet engine: compute score from config, enforce VEX-only quiet rules, emit inputs + `quietedBy` metadata in policy verdicts. | `/reports` policy result includes score, inputs, configVersion, quiet provenance; unit/integration tests prove reproducibility. |
| POLICY-CORE-09-006 | TODO | Policy Guild | POLICY-CORE-09-005, FEEDCORE-ENGINE-07-003 | Track unknown states with deterministic confidence bands that decay over time; expose state in policy outputs and docs. | Unknown flags + confidence band persisted, decay job deterministic, preview/report APIs show state with tests covering decay math. |
## Notes
- 2025-10-18: POLICY-CORE-09-001 completed. Binder + diagnostics + CLI scaffolding landed with tests; schema embedded at `src/StellaOps.Policy/Schemas/policy-schema@1.json` and referenced by docs/11_DATA_SCHEMAS.md.
- 2025-10-18: POLICY-CORE-09-002 completed. Snapshot store + audit trail implemented with deterministic digest hashing and tests covering revision increments and dedupe.
- 2025-10-18: POLICY-CORE-09-003 delivered. Preview service evaluates policy projections vs. baseline, returns verdict diffs, and ships with unit coverage.

View File

@@ -0,0 +1,81 @@
using System.Text.Json;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Serialization;
using StellaOps.Scanner.Core.Utility;
using Xunit;
namespace StellaOps.Scanner.Core.Tests.Contracts;
public sealed class ScanJobTests
{
[Fact]
public void SerializeAndDeserialize_RoundTripsDeterministically()
{
var createdAt = new DateTimeOffset(2025, 10, 18, 14, 30, 15, TimeSpan.Zero);
var jobId = ScannerIdentifiers.CreateJobId("registry.example.com/stellaops/scanner:1.2.3", "sha256:ABCDEF", "tenant-a", "request-1");
var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, "enqueue");
var error = new ScannerError(
ScannerErrorCode.AnalyzerFailure,
ScannerErrorSeverity.Error,
"Analyzer crashed for layer sha256:abc",
createdAt,
retryable: false,
details: new Dictionary<string, string>
{
["stage"] = "analyze-os",
["layer"] = "sha256:abc"
});
var job = new ScanJob(
jobId,
ScanJobStatus.Running,
"registry.example.com/stellaops/scanner:1.2.3",
"SHA256:ABCDEF",
createdAt,
createdAt,
correlationId,
"tenant-a",
new Dictionary<string, string>
{
["requestId"] = "request-1"
},
error);
var json = JsonSerializer.Serialize(job, ScannerJsonOptions.CreateDefault());
var deserialized = JsonSerializer.Deserialize<ScanJob>(json, ScannerJsonOptions.CreateDefault());
Assert.NotNull(deserialized);
Assert.Equal(job.Id, deserialized!.Id);
Assert.Equal(job.ImageDigest, deserialized.ImageDigest);
Assert.Equal(job.CorrelationId, deserialized.CorrelationId);
Assert.Equal(job.Metadata["requestId"], deserialized.Metadata["requestId"]);
var secondJson = JsonSerializer.Serialize(deserialized, ScannerJsonOptions.CreateDefault());
Assert.Equal(json, secondJson);
}
[Fact]
public void WithStatus_UpdatesTimestampDeterministically()
{
var createdAt = new DateTimeOffset(2025, 10, 18, 14, 30, 15, 123, TimeSpan.Zero);
var jobId = ScannerIdentifiers.CreateJobId("example/scanner:latest", "sha256:def", null, null);
var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, "enqueue");
var job = new ScanJob(
jobId,
ScanJobStatus.Pending,
"example/scanner:latest",
"sha256:def",
createdAt,
null,
correlationId,
null,
null,
null);
var updated = job.WithStatus(ScanJobStatus.Running, createdAt.AddSeconds(5));
Assert.Equal(ScanJobStatus.Running, updated.Status);
Assert.Equal(ScannerTimestamps.Normalize(createdAt.AddSeconds(5)), updated.UpdatedAt);
}
}

View File

@@ -0,0 +1,39 @@
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Observability;
using StellaOps.Scanner.Core.Utility;
using Xunit;
namespace StellaOps.Scanner.Core.Tests.Observability;
public sealed class ScannerLogExtensionsTests
{
[Fact]
public void BeginScanScope_PopulatesCorrelationContext()
{
using var factory = LoggerFactory.Create(builder => builder.AddFilter(_ => true));
var logger = factory.CreateLogger("test");
var jobId = ScannerIdentifiers.CreateJobId("example/scanner:1.0", "sha256:abc", null, null);
var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, "enqueue");
var job = new ScanJob(
jobId,
ScanJobStatus.Pending,
"example/scanner:1.0",
"sha256:abc",
DateTimeOffset.UtcNow,
null,
correlationId,
null,
null,
null);
using (logger.BeginScanScope(job, "enqueue"))
{
Assert.True(ScannerCorrelationContextAccessor.TryGetCorrelationId(out var current));
Assert.Equal(correlationId, current);
}
Assert.False(ScannerCorrelationContextAccessor.TryGetCorrelationId(out _));
}
}

View File

@@ -0,0 +1,89 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using Microsoft.IdentityModel.Tokens;
using StellaOps.Auth.Client;
using StellaOps.Scanner.Core.Security;
using Xunit;
namespace StellaOps.Scanner.Core.Tests.Security;
public sealed class AuthorityTokenSourceTests
{
[Fact]
public async Task GetAsync_ReusesCachedTokenUntilRefreshSkew()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero));
var client = new FakeTokenClient(timeProvider);
var source = new AuthorityTokenSource(client, TimeSpan.FromSeconds(30), timeProvider, NullLogger<AuthorityTokenSource>.Instance);
var token1 = await source.GetAsync("scanner", new[] { "scanner.read" });
Assert.Equal(1, client.RequestCount);
var token2 = await source.GetAsync("scanner", new[] { "scanner.read" });
Assert.Equal(1, client.RequestCount);
Assert.Equal(token1.AccessToken, token2.AccessToken);
timeProvider.Advance(TimeSpan.FromMinutes(3));
var token3 = await source.GetAsync("scanner", new[] { "scanner.read" });
Assert.Equal(2, client.RequestCount);
Assert.NotEqual(token1.AccessToken, token3.AccessToken);
}
[Fact]
public async Task InvalidateAsync_RemovesCachedToken()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero));
var client = new FakeTokenClient(timeProvider);
var source = new AuthorityTokenSource(client, TimeSpan.FromSeconds(30), timeProvider, NullLogger<AuthorityTokenSource>.Instance);
_ = await source.GetAsync("scanner", new[] { "scanner.read" });
Assert.Equal(1, client.RequestCount);
await source.InvalidateAsync("scanner", new[] { "scanner.read" });
_ = await source.GetAsync("scanner", new[] { "scanner.read" });
Assert.Equal(2, client.RequestCount);
}
private sealed class FakeTokenClient : IStellaOpsTokenClient
{
private readonly FakeTimeProvider timeProvider;
private int counter;
public FakeTokenClient(FakeTimeProvider timeProvider)
{
this.timeProvider = timeProvider;
}
public int RequestCount => counter;
public Task<StellaOpsTokenResult> RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default)
{
var access = $"token-{Interlocked.Increment(ref counter)}";
var expires = timeProvider.GetUtcNow().AddMinutes(2);
var scopes = scope is null
? Array.Empty<string>()
: scope.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
return Task.FromResult(new StellaOpsTokenResult(access, "Bearer", expires, scopes));
}
public Task<StellaOpsTokenResult> RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default)
=> throw new NotSupportedException();
public Task<JsonWebKeySet> GetJsonWebKeySetAsync(CancellationToken cancellationToken = default)
=> throw new NotSupportedException();
public ValueTask<StellaOpsTokenCacheEntry?> GetCachedTokenAsync(string key, CancellationToken cancellationToken = default)
=> ValueTask.FromResult<StellaOpsTokenCacheEntry?>(null);
public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default)
=> ValueTask.CompletedTask;
public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default)
=> ValueTask.CompletedTask;
}
}

View File

@@ -0,0 +1,117 @@
using System.Collections.Generic;
using System.IdentityModel.Tokens.Jwt;
using System.Security.Cryptography;
using Microsoft.Extensions.Time.Testing;
using Microsoft.Extensions.Options;
using Microsoft.IdentityModel.Tokens;
using StellaOps.Scanner.Core.Security;
using Xunit;
namespace StellaOps.Scanner.Core.Tests.Security;
public sealed class DpopProofValidatorTests
{
[Fact]
public async Task ValidateAsync_ReturnsSuccess_ForValidProof()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero));
var validator = new DpopProofValidator(Options.Create(new DpopValidationOptions()), new InMemoryDpopReplayCache(timeProvider), timeProvider);
using var key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
var securityKey = new ECDsaSecurityKey(key) { KeyId = Guid.NewGuid().ToString("N") };
var proof = CreateProof(timeProvider, securityKey, "GET", new Uri("https://scanner.example.com/api/v1/scans"));
var result = await validator.ValidateAsync(proof, "GET", new Uri("https://scanner.example.com/api/v1/scans"));
Assert.True(result.IsValid);
Assert.NotNull(result.PublicKey);
Assert.NotNull(result.JwtId);
}
[Fact]
public async Task ValidateAsync_Fails_OnNonceMismatch()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero));
var validator = new DpopProofValidator(Options.Create(new DpopValidationOptions()), new InMemoryDpopReplayCache(timeProvider), timeProvider);
using var key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
var securityKey = new ECDsaSecurityKey(key) { KeyId = Guid.NewGuid().ToString("N") };
var proof = CreateProof(timeProvider, securityKey, "POST", new Uri("https://scanner.example.com/api/v1/scans"), nonce: "expected");
var result = await validator.ValidateAsync(proof, "POST", new Uri("https://scanner.example.com/api/v1/scans"), nonce: "different");
Assert.False(result.IsValid);
Assert.Equal("invalid_token", result.ErrorCode);
}
[Fact]
public async Task ValidateAsync_Fails_OnReplay()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero));
var cache = new InMemoryDpopReplayCache(timeProvider);
var validator = new DpopProofValidator(Options.Create(new DpopValidationOptions()), cache, timeProvider);
using var key = ECDsa.Create(ECCurve.NamedCurves.nistP256);
var securityKey = new ECDsaSecurityKey(key) { KeyId = Guid.NewGuid().ToString("N") };
var jti = Guid.NewGuid().ToString();
var proof = CreateProof(timeProvider, securityKey, "GET", new Uri("https://scanner.example.com/api/v1/scans"), jti: jti);
var first = await validator.ValidateAsync(proof, "GET", new Uri("https://scanner.example.com/api/v1/scans"));
Assert.True(first.IsValid);
var second = await validator.ValidateAsync(proof, "GET", new Uri("https://scanner.example.com/api/v1/scans"));
Assert.False(second.IsValid);
Assert.Equal("replay", second.ErrorCode);
}
private static string CreateProof(FakeTimeProvider timeProvider, ECDsaSecurityKey key, string method, Uri uri, string? nonce = null, string? jti = null)
{
var handler = new JwtSecurityTokenHandler();
var signingCredentials = new SigningCredentials(key, SecurityAlgorithms.EcdsaSha256);
var jwk = JsonWebKeyConverter.ConvertFromECDsaSecurityKey(key);
var header = new JwtHeader(signingCredentials)
{
["typ"] = "dpop+jwt",
["jwk"] = new Dictionary<string, object?>
{
["kty"] = jwk.Kty,
["crv"] = jwk.Crv,
["x"] = jwk.X,
["y"] = jwk.Y
}
};
var payload = new JwtPayload
{
["htm"] = method.ToUpperInvariant(),
["htu"] = Normalize(uri),
["iat"] = timeProvider.GetUtcNow().ToUnixTimeSeconds(),
["jti"] = jti ?? Guid.NewGuid().ToString()
};
if (nonce is not null)
{
payload["nonce"] = nonce;
}
var token = new JwtSecurityToken(header, payload);
return handler.WriteToken(token);
}
private static string Normalize(Uri uri)
{
var builder = new UriBuilder(uri)
{
Fragment = string.Empty
};
builder.Host = builder.Host.ToLowerInvariant();
builder.Scheme = builder.Scheme.ToLowerInvariant();
if ((builder.Scheme == "http" && builder.Port == 80) || (builder.Scheme == "https" && builder.Port == 443))
{
builder.Port = -1;
}
return builder.Uri.GetComponents(UriComponents.SchemeAndServer | UriComponents.PathAndQuery, UriFormat.UriEscaped);
}
}

View File

@@ -0,0 +1,26 @@
using System;
using StellaOps.Scanner.Core.Security;
using Xunit;
namespace StellaOps.Scanner.Core.Tests.Security;
public sealed class RestartOnlyPluginGuardTests
{
[Fact]
public void EnsureRegistrationAllowed_AllowsNewPluginsBeforeSeal()
{
var guard = new RestartOnlyPluginGuard();
guard.EnsureRegistrationAllowed("./plugins/analyzer.dll");
Assert.Contains(guard.KnownPlugins, path => path.EndsWith("analyzer.dll", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void EnsureRegistrationAllowed_ThrowsAfterSeal()
{
var guard = new RestartOnlyPluginGuard(new[] { "./plugins/a.dll" });
guard.Seal();
Assert.Throws<InvalidOperationException>(() => guard.EnsureRegistrationAllowed("./plugins/new.dll"));
}
}

View File

@@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj" />
<ProjectReference Include="../StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOps.Auth.Abstractions.csproj" />
<ProjectReference Include="../StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,33 @@
using StellaOps.Scanner.Core.Utility;
using Xunit;
namespace StellaOps.Scanner.Core.Tests.Utility;
public sealed class ScannerIdentifiersTests
{
[Fact]
public void CreateJobId_IsDeterministicAndCaseInsensitive()
{
var first = ScannerIdentifiers.CreateJobId("registry.example.com/repo:latest", "SHA256:ABC", "Tenant-A", "salt");
var second = ScannerIdentifiers.CreateJobId("REGISTRY.EXAMPLE.COM/REPO:latest", "sha256:abc", "tenant-a", "salt");
Assert.Equal(first, second);
}
[Fact]
public void CreateDeterministicHash_ProducesLowercaseHex()
{
var hash = ScannerIdentifiers.CreateDeterministicHash("scan", "abc", "123");
Assert.Matches("^[0-9a-f]{64}$", hash);
Assert.Equal(hash, hash.ToLowerInvariant());
}
[Fact]
public void NormalizeImageReference_LowercasesRegistryAndRepository()
{
var normalized = ScannerIdentifiers.NormalizeImageReference("Registry.Example.com/StellaOps/Scanner:1.0");
Assert.Equal("registry.example.com/stellaops/scanner:1.0", normalized);
}
}

View File

@@ -0,0 +1,26 @@
using StellaOps.Scanner.Core.Utility;
using Xunit;
namespace StellaOps.Scanner.Core.Tests.Utility;
public sealed class ScannerTimestampsTests
{
[Fact]
public void Normalize_TrimsToMicroseconds()
{
var value = new DateTimeOffset(2025, 10, 18, 14, 30, 15, TimeSpan.Zero).AddTicks(7);
var normalized = ScannerTimestamps.Normalize(value);
var expectedTicks = value.UtcTicks - (value.UtcTicks % 10);
Assert.Equal(expectedTicks, normalized.UtcTicks);
}
[Fact]
public void ToIso8601_ProducesUtcString()
{
var value = new DateTimeOffset(2025, 10, 18, 14, 30, 15, TimeSpan.FromHours(-4));
var iso = ScannerTimestamps.ToIso8601(value);
Assert.Equal("2025-10-18T18:30:15.000000Z", iso);
}
}

View File

@@ -0,0 +1,29 @@
# AGENTS
## Role
Provide shared scanner contracts, observability primitives, and security utilities consumed by the WebService, Worker, analyzers, and downstream tooling.
## Scope
- Canonical DTOs for scan jobs, progress, outcomes, and error taxonomy shared across scanner services.
- Deterministic ID and timestamp helpers to guarantee reproducible job identifiers and ISO-8601 rendering.
- Observability helpers (logging scopes, correlation IDs, metric naming, activity sources) with negligible overhead.
- Authority/OpTok integrations, DPoP validation helpers, and restart-time plug-in guardrails for scanner components.
## Participants
- Scanner.WebService and Scanner.Worker depend on these primitives for request handling, queue interactions, and diagnostics.
- Policy/Signer integrations rely on deterministic identifiers and timestamps emitted here.
- DevOps/Offline kits bundle plug-in manifests validated via the guardrails defined in this module.
## Interfaces & contracts
- DTOs must round-trip via System.Text.Json with `JsonSerializerDefaults.Web` and preserve ordering.
- Deterministic helpers must not depend on ambient time/randomness; they derive IDs from explicit inputs and normalize timestamps to microsecond precision in UTC.
- Observability scopes expose `scanId`, `jobId`, `correlationId`, and `imageDigest` fields with `stellaops scanner` metric prefixing.
- Security helpers expose `IAuthorityTokenSource`, `IDPoPProofValidator`, and `IPluginCatalogGuard` abstractions with DI-friendly implementations.
## In/Out of scope
In: shared contracts, telemetry primitives, security utilities, plug-in manifest checks.
Out: queue implementations, analyzer logic, storage adapters, HTTP endpoints, UI wiring.
## Observability & security expectations
- No network calls except via registered Authority clients.
- Avoid allocations in hot paths; prefer struct enumerables/`ValueTask`.
- All logs structured, correlation IDs propagated, no secrets persisted.
- DPoP validation enforces algorithm allowlist (ES256/ES384) and ensures replay cache hooks.
## Tests
- `../StellaOps.Scanner.Core.Tests` owns unit coverage with deterministic fixtures.
- Golden JSON for DTO round-trips stored under `Fixtures/`.
- Security and observability helpers must include tests proving deterministic outputs and rejecting malformed proofs.

View File

@@ -0,0 +1,173 @@
using System.Collections.ObjectModel;
using System.Globalization;
using System.Text.Json.Serialization;
using StellaOps.Scanner.Core.Utility;
namespace StellaOps.Scanner.Core.Contracts;
[JsonConverter(typeof(ScanJobIdJsonConverter))]
public readonly record struct ScanJobId(Guid Value)
{
public static readonly ScanJobId Empty = new(Guid.Empty);
public override string ToString()
=> Value.ToString("n", CultureInfo.InvariantCulture);
public static ScanJobId From(Guid value)
=> new(value);
public static bool TryParse(string? text, out ScanJobId id)
{
if (Guid.TryParse(text, out var guid))
{
id = new ScanJobId(guid);
return true;
}
id = Empty;
return false;
}
}
[JsonConverter(typeof(JsonStringEnumConverter<ScanJobStatus>))]
public enum ScanJobStatus
{
Unknown = 0,
Pending,
Queued,
Running,
Succeeded,
Failed,
Cancelled
}
public sealed class ScanJob
{
private static readonly IReadOnlyDictionary<string, string> EmptyMetadata =
new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(0, StringComparer.Ordinal));
[JsonConstructor]
public ScanJob(
ScanJobId id,
ScanJobStatus status,
string imageReference,
string? imageDigest,
DateTimeOffset createdAt,
DateTimeOffset? updatedAt,
string correlationId,
string? tenantId,
IReadOnlyDictionary<string, string>? metadata = null,
ScannerError? failure = null)
{
if (string.IsNullOrWhiteSpace(imageReference))
{
throw new ArgumentException("Image reference cannot be null or whitespace.", nameof(imageReference));
}
if (string.IsNullOrWhiteSpace(correlationId))
{
throw new ArgumentException("Correlation identifier cannot be null or whitespace.", nameof(correlationId));
}
Id = id;
Status = status;
ImageReference = imageReference.Trim();
ImageDigest = NormalizeDigest(imageDigest);
CreatedAt = ScannerTimestamps.Normalize(createdAt);
UpdatedAt = updatedAt is null ? null : ScannerTimestamps.Normalize(updatedAt.Value);
CorrelationId = correlationId;
TenantId = string.IsNullOrWhiteSpace(tenantId) ? null : tenantId.Trim();
Metadata = metadata is null or { Count: 0 }
? EmptyMetadata
: new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(metadata, StringComparer.Ordinal));
Failure = failure;
}
[JsonPropertyName("id")]
[JsonPropertyOrder(0)]
public ScanJobId Id { get; }
[JsonPropertyName("status")]
[JsonPropertyOrder(1)]
public ScanJobStatus Status { get; init; }
[JsonPropertyName("imageReference")]
[JsonPropertyOrder(2)]
public string ImageReference { get; }
[JsonPropertyName("imageDigest")]
[JsonPropertyOrder(3)]
public string? ImageDigest { get; }
[JsonPropertyName("createdAt")]
[JsonPropertyOrder(4)]
public DateTimeOffset CreatedAt { get; }
[JsonPropertyName("updatedAt")]
[JsonPropertyOrder(5)]
public DateTimeOffset? UpdatedAt { get; init; }
[JsonPropertyName("correlationId")]
[JsonPropertyOrder(6)]
public string CorrelationId { get; }
[JsonPropertyName("tenantId")]
[JsonPropertyOrder(7)]
public string? TenantId { get; }
[JsonPropertyName("metadata")]
[JsonPropertyOrder(8)]
public IReadOnlyDictionary<string, string> Metadata { get; }
[JsonPropertyName("failure")]
[JsonPropertyOrder(9)]
public ScannerError? Failure { get; init; }
public ScanJob WithStatus(ScanJobStatus status, DateTimeOffset? updatedAt = null)
=> new(
Id,
status,
ImageReference,
ImageDigest,
CreatedAt,
updatedAt ?? UpdatedAt ?? CreatedAt,
CorrelationId,
TenantId,
Metadata,
Failure);
public ScanJob WithFailure(ScannerError failure, DateTimeOffset? updatedAt = null, TimeProvider? timeProvider = null)
=> new(
Id,
ScanJobStatus.Failed,
ImageReference,
ImageDigest,
CreatedAt,
updatedAt ?? ScannerTimestamps.UtcNow(timeProvider),
CorrelationId,
TenantId,
Metadata,
failure);
private static string? NormalizeDigest(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return null;
}
var trimmed = digest.Trim();
if (!trimmed.StartsWith("sha", StringComparison.OrdinalIgnoreCase))
{
return trimmed;
}
var parts = trimmed.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length != 2)
{
return trimmed.ToLowerInvariant();
}
return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}";
}
}

View File

@@ -0,0 +1,26 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Core.Contracts;
internal sealed class ScanJobIdJsonConverter : JsonConverter<ScanJobId>
{
public override ScanJobId Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType != JsonTokenType.String)
{
throw new JsonException("Expected scan job identifier to be a string.");
}
var value = reader.GetString();
if (!ScanJobId.TryParse(value, out var id))
{
throw new JsonException("Invalid scan job identifier.");
}
return id;
}
public override void Write(Utf8JsonWriter writer, ScanJobId value, JsonSerializerOptions options)
=> writer.WriteStringValue(value.ToString());
}

View File

@@ -0,0 +1,121 @@
using System.Collections.ObjectModel;
using System.Text.Json.Serialization;
using StellaOps.Scanner.Core.Utility;
namespace StellaOps.Scanner.Core.Contracts;
[JsonConverter(typeof(JsonStringEnumConverter<ScanStage>))]
public enum ScanStage
{
Unknown = 0,
ResolveImage,
FetchLayers,
MountLayers,
AnalyzeOperatingSystem,
AnalyzeLanguageEcosystems,
AnalyzeNativeArtifacts,
ComposeSbom,
BuildDiffs,
EmitArtifacts,
SignArtifacts,
Complete
}
[JsonConverter(typeof(JsonStringEnumConverter<ScanProgressEventKind>))]
public enum ScanProgressEventKind
{
Progress = 0,
StageStarted,
StageCompleted,
Warning,
Error
}
public sealed class ScanProgressEvent
{
private static readonly IReadOnlyDictionary<string, string> EmptyAttributes =
new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(0, StringComparer.Ordinal));
[JsonConstructor]
public ScanProgressEvent(
ScanJobId jobId,
ScanStage stage,
ScanProgressEventKind kind,
int sequence,
DateTimeOffset timestamp,
double? percentComplete = null,
string? message = null,
IReadOnlyDictionary<string, string>? attributes = null,
ScannerError? error = null)
{
if (sequence < 0)
{
throw new ArgumentOutOfRangeException(nameof(sequence), sequence, "Sequence cannot be negative.");
}
JobId = jobId;
Stage = stage;
Kind = kind;
Sequence = sequence;
Timestamp = ScannerTimestamps.Normalize(timestamp);
PercentComplete = percentComplete is < 0 or > 100 ? null : percentComplete;
Message = message is { Length: > 0 } ? message.Trim() : null;
Attributes = attributes is null or { Count: 0 }
? EmptyAttributes
: new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(attributes, StringComparer.Ordinal));
Error = error;
}
[JsonPropertyName("jobId")]
[JsonPropertyOrder(0)]
public ScanJobId JobId { get; }
[JsonPropertyName("stage")]
[JsonPropertyOrder(1)]
public ScanStage Stage { get; }
[JsonPropertyName("kind")]
[JsonPropertyOrder(2)]
public ScanProgressEventKind Kind { get; }
[JsonPropertyName("sequence")]
[JsonPropertyOrder(3)]
public int Sequence { get; }
[JsonPropertyName("timestamp")]
[JsonPropertyOrder(4)]
public DateTimeOffset Timestamp { get; }
[JsonPropertyName("percentComplete")]
[JsonPropertyOrder(5)]
public double? PercentComplete { get; }
[JsonPropertyName("message")]
[JsonPropertyOrder(6)]
public string? Message { get; }
[JsonPropertyName("attributes")]
[JsonPropertyOrder(7)]
public IReadOnlyDictionary<string, string> Attributes { get; }
[JsonPropertyName("error")]
[JsonPropertyOrder(8)]
public ScannerError? Error { get; }
public ScanProgressEvent With(
ScanProgressEventKind? kind = null,
double? percentComplete = null,
string? message = null,
IReadOnlyDictionary<string, string>? attributes = null,
ScannerError? error = null)
=> new(
JobId,
Stage,
kind ?? Kind,
Sequence,
Timestamp,
percentComplete ?? PercentComplete,
message ?? Message,
attributes ?? Attributes,
error ?? Error);
}

View File

@@ -0,0 +1,110 @@
using System.Collections.ObjectModel;
using System.Text.Json.Serialization;
using StellaOps.Scanner.Core.Utility;
namespace StellaOps.Scanner.Core.Contracts;
[JsonConverter(typeof(JsonStringEnumConverter<ScannerErrorCode>))]
public enum ScannerErrorCode
{
Unknown = 0,
InvalidImageReference,
ImageNotFound,
AuthorizationFailed,
QueueUnavailable,
StorageUnavailable,
AnalyzerFailure,
ExportFailure,
SigningFailure,
RuntimeFailure,
Timeout,
Cancelled,
PluginViolation
}
[JsonConverter(typeof(JsonStringEnumConverter<ScannerErrorSeverity>))]
public enum ScannerErrorSeverity
{
Warning = 0,
Error,
Fatal
}
public sealed class ScannerError
{
private static readonly IReadOnlyDictionary<string, string> EmptyDetails =
new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(0, StringComparer.Ordinal));
[JsonConstructor]
public ScannerError(
ScannerErrorCode code,
ScannerErrorSeverity severity,
string message,
DateTimeOffset timestamp,
bool retryable,
IReadOnlyDictionary<string, string>? details = null,
string? stage = null,
string? component = null)
{
if (string.IsNullOrWhiteSpace(message))
{
throw new ArgumentException("Error message cannot be null or whitespace.", nameof(message));
}
Code = code;
Severity = severity;
Message = message.Trim();
Timestamp = ScannerTimestamps.Normalize(timestamp);
Retryable = retryable;
Stage = stage;
Component = component;
Details = details is null or { Count: 0 }
? EmptyDetails
: new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(details, StringComparer.Ordinal));
}
[JsonPropertyName("code")]
[JsonPropertyOrder(0)]
public ScannerErrorCode Code { get; }
[JsonPropertyName("severity")]
[JsonPropertyOrder(1)]
public ScannerErrorSeverity Severity { get; }
[JsonPropertyName("message")]
[JsonPropertyOrder(2)]
public string Message { get; }
[JsonPropertyName("timestamp")]
[JsonPropertyOrder(3)]
public DateTimeOffset Timestamp { get; }
[JsonPropertyName("retryable")]
[JsonPropertyOrder(4)]
public bool Retryable { get; }
[JsonPropertyName("stage")]
[JsonPropertyOrder(5)]
public string? Stage { get; }
[JsonPropertyName("component")]
[JsonPropertyOrder(6)]
public string? Component { get; }
[JsonPropertyName("details")]
[JsonPropertyOrder(7)]
public IReadOnlyDictionary<string, string> Details { get; }
public ScannerError WithDetail(string key, string value)
{
ArgumentException.ThrowIfNullOrWhiteSpace(key);
ArgumentException.ThrowIfNullOrWhiteSpace(value);
var mutable = new Dictionary<string, string>(Details, StringComparer.Ordinal)
{
[key] = value
};
return new ScannerError(Code, Severity, Message, Timestamp, Retryable, mutable, Stage, Component);
}
}

View File

@@ -0,0 +1,80 @@
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Utility;
namespace StellaOps.Scanner.Core.Observability;
public readonly record struct ScannerCorrelationContext(
ScanJobId JobId,
string CorrelationId,
string? Stage,
string? Component,
string? Audience = null)
{
public static ScannerCorrelationContext Create(
ScanJobId jobId,
string? stage = null,
string? component = null,
string? audience = null)
{
var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, stage, component);
return new ScannerCorrelationContext(jobId, correlationId, stage, component, audience);
}
public string DeterministicHash()
=> ScannerIdentifiers.CreateDeterministicHash(
JobId.ToString(),
Stage ?? string.Empty,
Component ?? string.Empty,
Audience ?? string.Empty);
}
public static class ScannerCorrelationContextAccessor
{
private static readonly AsyncLocal<ScannerCorrelationContext?> CurrentContext = new();
public static ScannerCorrelationContext? Current => CurrentContext.Value;
public static IDisposable Push(in ScannerCorrelationContext context)
{
var previous = CurrentContext.Value;
CurrentContext.Value = context;
return new DisposableScope(() => CurrentContext.Value = previous);
}
public static bool TryGetCorrelationId([NotNullWhen(true)] out string? correlationId)
{
var context = CurrentContext.Value;
if (context.HasValue)
{
correlationId = context.Value.CorrelationId;
return true;
}
correlationId = null;
return false;
}
private sealed class DisposableScope : IDisposable
{
private readonly Action release;
private bool disposed;
public DisposableScope(Action release)
{
this.release = release ?? throw new ArgumentNullException(nameof(release));
}
public void Dispose()
{
if (disposed)
{
return;
}
disposed = true;
release();
}
}
}

View File

@@ -0,0 +1,55 @@
using System.Diagnostics;
using System.Diagnostics.Metrics;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Utility;
namespace StellaOps.Scanner.Core.Observability;
public static class ScannerDiagnostics
{
public const string ActivitySourceName = "StellaOps.Scanner";
public const string ActivityVersion = "1.0.0";
public const string MeterName = "stellaops.scanner";
public const string MeterVersion = "1.0.0";
public static ActivitySource ActivitySource { get; } = new(ActivitySourceName, ActivityVersion);
public static Meter Meter { get; } = new(MeterName, MeterVersion);
public static Activity? StartActivity(
string name,
ScanJobId jobId,
string? stage = null,
string? component = null,
ActivityKind kind = ActivityKind.Internal,
IEnumerable<KeyValuePair<string, object?>>? tags = null)
{
var activity = ActivitySource.StartActivity(name, kind);
if (activity is null)
{
return null;
}
activity.SetTag("stellaops.scanner.job_id", jobId.ToString());
activity.SetTag("stellaops.scanner.correlation_id", ScannerIdentifiers.CreateCorrelationId(jobId, stage, component));
if (!string.IsNullOrWhiteSpace(stage))
{
activity.SetTag("stellaops.scanner.stage", stage);
}
if (!string.IsNullOrWhiteSpace(component))
{
activity.SetTag("stellaops.scanner.component", component);
}
if (tags is not null)
{
foreach (var tag in tags)
{
activity?.SetTag(tag.Key, tag.Value);
}
}
return activity;
}
}

View File

@@ -0,0 +1,115 @@
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Utility;
namespace StellaOps.Scanner.Core.Observability;
public static class ScannerLogExtensions
{
private sealed class NoopScope : IDisposable
{
public static NoopScope Instance { get; } = new();
public void Dispose()
{
}
}
private sealed class CompositeScope : IDisposable
{
private readonly IDisposable first;
private readonly IDisposable second;
private bool disposed;
public CompositeScope(IDisposable first, IDisposable second)
{
this.first = first;
this.second = second;
}
public void Dispose()
{
if (disposed)
{
return;
}
disposed = true;
second.Dispose();
first.Dispose();
}
}
public static IDisposable BeginScanScope(this ILogger? logger, ScanJob job, string? stage = null, string? component = null)
{
var correlation = ScannerCorrelationContext.Create(job.Id, stage, component);
var logScope = logger is null
? NoopScope.Instance
: logger.BeginScope(CreateScopeState(
job.Id,
job.CorrelationId,
stage,
component,
job.TenantId,
job.ImageDigest)) ?? NoopScope.Instance;
var correlationScope = ScannerCorrelationContextAccessor.Push(correlation);
return new CompositeScope(logScope, correlationScope);
}
public static IDisposable BeginProgressScope(this ILogger? logger, ScanProgressEvent progress, string? component = null)
{
var correlationId = ScannerIdentifiers.CreateCorrelationId(progress.JobId, progress.Stage.ToString(), component);
var correlation = new ScannerCorrelationContext(progress.JobId, correlationId, progress.Stage.ToString(), component);
var logScope = logger is null
? NoopScope.Instance
: logger.BeginScope(new Dictionary<string, object?>(6, StringComparer.Ordinal)
{
["scanId"] = progress.JobId.ToString(),
["stage"] = progress.Stage.ToString(),
["sequence"] = progress.Sequence,
["kind"] = progress.Kind.ToString(),
["correlationId"] = correlationId,
["component"] = component ?? string.Empty
}) ?? NoopScope.Instance;
var correlationScope = ScannerCorrelationContextAccessor.Push(correlation);
return new CompositeScope(logScope, correlationScope);
}
public static IDisposable BeginCorrelationScope(this ILogger? logger, ScannerCorrelationContext context)
{
var scope = logger is null
? NoopScope.Instance
: logger.BeginScope(CreateScopeState(context.JobId, context.CorrelationId, context.Stage, context.Component, null, null)) ?? NoopScope.Instance;
var correlationScope = ScannerCorrelationContextAccessor.Push(context);
return new CompositeScope(scope, correlationScope);
}
private static Dictionary<string, object?> CreateScopeState(
ScanJobId jobId,
string correlationId,
string? stage,
string? component,
string? tenantId,
string? imageDigest)
{
var state = new Dictionary<string, object?>(6, StringComparer.Ordinal)
{
["scanId"] = jobId.ToString(),
["correlationId"] = correlationId,
["stage"] = stage ?? string.Empty,
["component"] = component ?? string.Empty,
["tenantId"] = tenantId ?? string.Empty
};
if (!string.IsNullOrEmpty(imageDigest))
{
state["imageDigest"] = imageDigest;
}
return state;
}
}

View File

@@ -0,0 +1,55 @@
using System.Collections.Frozen;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Utility;
namespace StellaOps.Scanner.Core.Observability;
public static class ScannerMetricNames
{
public const string Prefix = "stellaops.scanner";
public const string QueueLatency = $"{Prefix}.queue.latency";
public const string QueueDepth = $"{Prefix}.queue.depth";
public const string StageDuration = $"{Prefix}.stage.duration";
public const string StageProgress = $"{Prefix}.stage.progress";
public const string JobCount = $"{Prefix}.jobs.count";
public const string JobFailures = $"{Prefix}.jobs.failures";
public const string ArtifactBytes = $"{Prefix}.artifacts.bytes";
public static FrozenDictionary<string, object?> BuildJobTags(ScanJob job, string? stage = null, string? component = null)
{
ArgumentNullException.ThrowIfNull(job);
var builder = new Dictionary<string, object?>(6, StringComparer.Ordinal)
{
["jobId"] = job.Id.ToString(),
["stage"] = stage ?? string.Empty,
["component"] = component ?? string.Empty,
["tenantId"] = job.TenantId ?? string.Empty,
["correlationId"] = job.CorrelationId,
["status"] = job.Status.ToString()
};
if (!string.IsNullOrEmpty(job.ImageDigest))
{
builder["imageDigest"] = job.ImageDigest;
}
return builder.ToFrozenDictionary(StringComparer.Ordinal);
}
public static FrozenDictionary<string, object?> BuildEventTags(ScanProgressEvent progress)
{
ArgumentNullException.ThrowIfNull(progress);
var builder = new Dictionary<string, object?>(5, StringComparer.Ordinal)
{
["jobId"] = progress.JobId.ToString(),
["stage"] = progress.Stage.ToString(),
["kind"] = progress.Kind.ToString(),
["sequence"] = progress.Sequence,
["correlationId"] = ScannerIdentifiers.CreateCorrelationId(progress.JobId, progress.Stage.ToString())
};
return builder.ToFrozenDictionary(StringComparer.Ordinal);
}
}

View File

@@ -0,0 +1,128 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Auth.Client;
using StellaOps.Scanner.Core.Utility;
namespace StellaOps.Scanner.Core.Security;
public sealed class AuthorityTokenSource : IAuthorityTokenSource
{
private readonly IStellaOpsTokenClient tokenClient;
private readonly TimeProvider timeProvider;
private readonly TimeSpan refreshSkew;
private readonly ILogger<AuthorityTokenSource>? logger;
private readonly ConcurrentDictionary<string, CacheEntry> cache = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, SemaphoreSlim> locks = new(StringComparer.Ordinal);
public AuthorityTokenSource(
IStellaOpsTokenClient tokenClient,
TimeSpan? refreshSkew = null,
TimeProvider? timeProvider = null,
ILogger<AuthorityTokenSource>? logger = null)
{
this.tokenClient = tokenClient ?? throw new ArgumentNullException(nameof(tokenClient));
this.timeProvider = timeProvider ?? TimeProvider.System;
this.logger = logger;
this.refreshSkew = refreshSkew is { } value && value > TimeSpan.Zero ? value : TimeSpan.FromSeconds(30);
}
public async ValueTask<ScannerOperationalToken> GetAsync(string audience, IEnumerable<string> scopes, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(audience);
var normalizedAudience = NormalizeAudience(audience);
var normalizedScopes = NormalizeScopes(scopes, normalizedAudience);
var cacheKey = BuildCacheKey(normalizedAudience, normalizedScopes);
if (cache.TryGetValue(cacheKey, out var cached) && !cached.Token.IsExpired(timeProvider, refreshSkew))
{
return cached.Token;
}
var mutex = locks.GetOrAdd(cacheKey, static _ => new SemaphoreSlim(1, 1));
await mutex.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (cache.TryGetValue(cacheKey, out cached) && !cached.Token.IsExpired(timeProvider, refreshSkew))
{
return cached.Token;
}
var scopeString = string.Join(' ', normalizedScopes);
var tokenResult = await tokenClient.RequestClientCredentialsTokenAsync(scopeString, cancellationToken).ConfigureAwait(false);
var token = ScannerOperationalToken.FromResult(
tokenResult.AccessToken,
tokenResult.TokenType,
tokenResult.ExpiresAtUtc,
tokenResult.Scopes);
cache[cacheKey] = new CacheEntry(token);
logger?.LogDebug(
"Issued new scanner OpTok for audience {Audience} with scopes {Scopes}; expires at {ExpiresAt}.",
normalizedAudience,
scopeString,
token.ExpiresAt);
return token;
}
finally
{
mutex.Release();
}
}
public ValueTask InvalidateAsync(string audience, IEnumerable<string> scopes, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(audience);
var normalizedAudience = NormalizeAudience(audience);
var normalizedScopes = NormalizeScopes(scopes, normalizedAudience);
var cacheKey = BuildCacheKey(normalizedAudience, normalizedScopes);
cache.TryRemove(cacheKey, out _);
if (locks.TryRemove(cacheKey, out var mutex))
{
mutex.Dispose();
}
logger?.LogDebug("Invalidated cached OpTok for {Audience} ({CacheKey}).", normalizedAudience, cacheKey);
return ValueTask.CompletedTask;
}
private static string NormalizeAudience(string audience)
=> audience.Trim().ToLowerInvariant();
private static IReadOnlyList<string> NormalizeScopes(IEnumerable<string> scopes, string audience)
{
var set = new SortedSet<string>(StringComparer.Ordinal)
{
$"aud:{audience}"
};
if (scopes is not null)
{
foreach (var scope in scopes)
{
if (string.IsNullOrWhiteSpace(scope))
{
continue;
}
set.Add(scope.Trim());
}
}
return set.ToArray();
}
private static string BuildCacheKey(string audience, IReadOnlyList<string> scopes)
=> ScannerIdentifiers.CreateDeterministicHash(audience, string.Join(' ', scopes));
private readonly record struct CacheEntry(ScannerOperationalToken Token);
}

View File

@@ -0,0 +1,248 @@
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.IdentityModel.Tokens;
using System.IdentityModel.Tokens.Jwt;
namespace StellaOps.Scanner.Core.Security;
public sealed class DpopProofValidator : IDpopProofValidator
{
private static readonly string ProofType = "dpop+jwt";
private readonly DpopValidationOptions options;
private readonly IDpopReplayCache replayCache;
private readonly TimeProvider timeProvider;
private readonly ILogger<DpopProofValidator>? logger;
private readonly JwtSecurityTokenHandler tokenHandler = new();
public DpopProofValidator(
IOptions<DpopValidationOptions> options,
IDpopReplayCache? replayCache = null,
TimeProvider? timeProvider = null,
ILogger<DpopProofValidator>? logger = null)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
var cloned = options.Value ?? throw new InvalidOperationException("DPoP options must be provided.");
cloned.Validate();
this.options = cloned;
this.replayCache = replayCache ?? NullReplayCache.Instance;
this.timeProvider = timeProvider ?? TimeProvider.System;
this.logger = logger;
}
public async ValueTask<DpopValidationResult> ValidateAsync(string proof, string httpMethod, Uri httpUri, string? nonce = null, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(proof);
ArgumentException.ThrowIfNullOrWhiteSpace(httpMethod);
ArgumentNullException.ThrowIfNull(httpUri);
var now = timeProvider.GetUtcNow();
if (!TryDecodeSegment(proof, segmentIndex: 0, out var headerElement, out var headerError))
{
logger?.LogWarning("DPoP header decode failure: {Error}", headerError);
return DpopValidationResult.Failure("invalid_header", headerError ?? "Unable to decode header.");
}
if (!headerElement.TryGetProperty("typ", out var typElement) || !string.Equals(typElement.GetString(), ProofType, StringComparison.OrdinalIgnoreCase))
{
return DpopValidationResult.Failure("invalid_header", "DPoP proof missing typ=dpop+jwt header.");
}
if (!headerElement.TryGetProperty("alg", out var algElement))
{
return DpopValidationResult.Failure("invalid_header", "DPoP proof missing alg header.");
}
var algorithm = algElement.GetString()?.Trim().ToUpperInvariant();
if (string.IsNullOrEmpty(algorithm) || !options.NormalizedAlgorithms.Contains(algorithm))
{
return DpopValidationResult.Failure("invalid_header", "Unsupported DPoP algorithm.");
}
if (!headerElement.TryGetProperty("jwk", out var jwkElement))
{
return DpopValidationResult.Failure("invalid_header", "DPoP proof missing jwk header.");
}
JsonWebKey jwk;
try
{
jwk = new JsonWebKey(jwkElement.GetRawText());
}
catch (Exception ex)
{
logger?.LogWarning(ex, "Failed to parse DPoP jwk header.");
return DpopValidationResult.Failure("invalid_header", "DPoP proof jwk header is invalid.");
}
if (!TryDecodeSegment(proof, segmentIndex: 1, out var payloadElement, out var payloadError))
{
logger?.LogWarning("DPoP payload decode failure: {Error}", payloadError);
return DpopValidationResult.Failure("invalid_payload", payloadError ?? "Unable to decode payload.");
}
if (!payloadElement.TryGetProperty("htm", out var htmElement))
{
return DpopValidationResult.Failure("invalid_payload", "DPoP proof missing htm claim.");
}
var method = httpMethod.Trim().ToUpperInvariant();
if (!string.Equals(htmElement.GetString(), method, StringComparison.Ordinal))
{
return DpopValidationResult.Failure("invalid_payload", "DPoP htm does not match request method.");
}
if (!payloadElement.TryGetProperty("htu", out var htuElement))
{
return DpopValidationResult.Failure("invalid_payload", "DPoP proof missing htu claim.");
}
var normalizedHtu = NormalizeHtu(httpUri);
if (!string.Equals(htuElement.GetString(), normalizedHtu, StringComparison.Ordinal))
{
return DpopValidationResult.Failure("invalid_payload", "DPoP htu does not match request URI.");
}
if (!payloadElement.TryGetProperty("iat", out var iatElement) || iatElement.ValueKind is not JsonValueKind.Number)
{
return DpopValidationResult.Failure("invalid_payload", "DPoP proof missing iat claim.");
}
if (!payloadElement.TryGetProperty("jti", out var jtiElement) || jtiElement.ValueKind != JsonValueKind.String)
{
return DpopValidationResult.Failure("invalid_payload", "DPoP proof missing jti claim.");
}
long iatSeconds;
try
{
iatSeconds = iatElement.GetInt64();
}
catch (Exception)
{
return DpopValidationResult.Failure("invalid_payload", "DPoP proof iat claim is not a valid number.");
}
var issuedAt = DateTimeOffset.FromUnixTimeSeconds(iatSeconds).ToUniversalTime();
if (issuedAt - options.AllowedClockSkew > now)
{
return DpopValidationResult.Failure("invalid_token", "DPoP proof issued in the future.");
}
if (now - issuedAt > options.ProofLifetime + options.AllowedClockSkew)
{
return DpopValidationResult.Failure("invalid_token", "DPoP proof expired.");
}
if (nonce is not null)
{
if (!payloadElement.TryGetProperty("nonce", out var nonceElement) || nonceElement.ValueKind != JsonValueKind.String)
{
return DpopValidationResult.Failure("invalid_token", "DPoP proof missing nonce claim.");
}
if (!string.Equals(nonceElement.GetString(), nonce, StringComparison.Ordinal))
{
return DpopValidationResult.Failure("invalid_token", "DPoP nonce mismatch.");
}
}
var jwtId = jtiElement.GetString()!;
try
{
var parameters = new TokenValidationParameters
{
ValidateAudience = false,
ValidateIssuer = false,
ValidateLifetime = false,
ValidateTokenReplay = false,
RequireSignedTokens = true,
ValidateIssuerSigningKey = true,
IssuerSigningKey = jwk,
ValidAlgorithms = options.NormalizedAlgorithms.ToArray()
};
tokenHandler.ValidateToken(proof, parameters, out _);
}
catch (Exception ex)
{
logger?.LogWarning(ex, "DPoP proof signature validation failed.");
return DpopValidationResult.Failure("invalid_signature", "DPoP proof signature validation failed.");
}
if (!await replayCache.TryStoreAsync(jwtId, issuedAt + options.ReplayWindow, cancellationToken).ConfigureAwait(false))
{
return DpopValidationResult.Failure("replay", "DPoP proof already used.");
}
return DpopValidationResult.Success(jwk, jwtId, issuedAt);
}
private static bool TryDecodeSegment(string token, int segmentIndex, out JsonElement element, out string? error)
{
element = default;
error = null;
var segments = token.Split('.');
if (segments.Length != 3)
{
error = "Token must contain three segments.";
return false;
}
if (segmentIndex < 0 || segmentIndex > 1)
{
error = "Segment index must be 0 or 1.";
return false;
}
try
{
var jsonBytes = Base64UrlEncoder.DecodeBytes(segments[segmentIndex]);
using var document = JsonDocument.Parse(jsonBytes);
element = document.RootElement.Clone();
return true;
}
catch (Exception ex)
{
error = ex.Message;
return false;
}
}
private static string NormalizeHtu(Uri uri)
{
var builder = new UriBuilder(uri)
{
Fragment = string.Empty
};
builder.Host = builder.Host.ToLowerInvariant();
builder.Scheme = builder.Scheme.ToLowerInvariant();
if ((builder.Scheme == "http" && builder.Port == 80) || (builder.Scheme == "https" && builder.Port == 443))
{
builder.Port = -1;
}
return builder.Uri.GetComponents(UriComponents.SchemeAndServer | UriComponents.PathAndQuery, UriFormat.UriEscaped);
}
private sealed class NullReplayCache : IDpopReplayCache
{
public static NullReplayCache Instance { get; } = new();
public ValueTask<bool> TryStoreAsync(string jwtId, DateTimeOffset expiresAt, CancellationToken cancellationToken = default)
=> ValueTask.FromResult(true);
}
}

View File

@@ -0,0 +1,58 @@
using System.Collections.Immutable;
using System.Linq;
namespace StellaOps.Scanner.Core.Security;
public sealed class DpopValidationOptions
{
private readonly HashSet<string> allowedAlgorithms = new(StringComparer.Ordinal);
public DpopValidationOptions()
{
allowedAlgorithms.Add("ES256");
allowedAlgorithms.Add("ES384");
}
public TimeSpan ProofLifetime { get; set; } = TimeSpan.FromMinutes(2);
public TimeSpan AllowedClockSkew { get; set; } = TimeSpan.FromSeconds(30);
public TimeSpan ReplayWindow { get; set; } = TimeSpan.FromMinutes(5);
public ISet<string> AllowedAlgorithms => allowedAlgorithms;
public IReadOnlySet<string> NormalizedAlgorithms { get; private set; } = ImmutableHashSet<string>.Empty;
public void Validate()
{
if (ProofLifetime <= TimeSpan.Zero)
{
throw new InvalidOperationException("DPoP proof lifetime must be greater than zero.");
}
if (AllowedClockSkew < TimeSpan.Zero || AllowedClockSkew > TimeSpan.FromMinutes(5))
{
throw new InvalidOperationException("DPoP allowed clock skew must be between 0 seconds and 5 minutes.");
}
if (ReplayWindow < TimeSpan.Zero)
{
throw new InvalidOperationException("DPoP replay window must be greater than or equal to zero.");
}
if (allowedAlgorithms.Count == 0)
{
throw new InvalidOperationException("At least one allowed DPoP algorithm must be configured.");
}
NormalizedAlgorithms = allowedAlgorithms
.Select(static algorithm => algorithm.Trim().ToUpperInvariant())
.Where(static algorithm => algorithm.Length > 0)
.ToImmutableHashSet(StringComparer.Ordinal);
if (NormalizedAlgorithms.Count == 0)
{
throw new InvalidOperationException("Allowed DPoP algorithms cannot be empty after normalization.");
}
}
}

View File

@@ -0,0 +1,34 @@
using Microsoft.IdentityModel.Tokens;
namespace StellaOps.Scanner.Core.Security;
public sealed class DpopValidationResult
{
private DpopValidationResult(bool success, string? errorCode, string? errorDescription, SecurityKey? key, string? jwtId, DateTimeOffset? issuedAt)
{
IsValid = success;
ErrorCode = errorCode;
ErrorDescription = errorDescription;
PublicKey = key;
JwtId = jwtId;
IssuedAt = issuedAt;
}
public bool IsValid { get; }
public string? ErrorCode { get; }
public string? ErrorDescription { get; }
public SecurityKey? PublicKey { get; }
public string? JwtId { get; }
public DateTimeOffset? IssuedAt { get; }
public static DpopValidationResult Success(SecurityKey key, string jwtId, DateTimeOffset issuedAt)
=> new(true, null, null, key, jwtId, issuedAt);
public static DpopValidationResult Failure(string code, string description)
=> new(false, code, description, null, null, null);
}

View File

@@ -0,0 +1,11 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Core.Security;
public interface IAuthorityTokenSource
{
ValueTask<ScannerOperationalToken> GetAsync(string audience, IEnumerable<string> scopes, CancellationToken cancellationToken = default);
ValueTask InvalidateAsync(string audience, IEnumerable<string> scopes, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Core.Security;
public interface IDpopProofValidator
{
ValueTask<DpopValidationResult> ValidateAsync(string proof, string httpMethod, Uri httpUri, string? nonce = null, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Core.Security;
public interface IDpopReplayCache
{
ValueTask<bool> TryStoreAsync(string jwtId, DateTimeOffset expiresAt, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,12 @@
namespace StellaOps.Scanner.Core.Security;
public interface IPluginCatalogGuard
{
IReadOnlyCollection<string> KnownPlugins { get; }
bool IsSealed { get; }
void EnsureRegistrationAllowed(string pluginPath);
void Seal();
}

View File

@@ -0,0 +1,65 @@
using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Core.Security;
public sealed class InMemoryDpopReplayCache : IDpopReplayCache
{
private readonly ConcurrentDictionary<string, DateTimeOffset> entries = new(StringComparer.Ordinal);
private readonly TimeProvider timeProvider;
public InMemoryDpopReplayCache(TimeProvider? timeProvider = null)
{
this.timeProvider = timeProvider ?? TimeProvider.System;
}
public ValueTask<bool> TryStoreAsync(string jwtId, DateTimeOffset expiresAt, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(jwtId);
var now = timeProvider.GetUtcNow();
RemoveExpired(now);
if (entries.TryAdd(jwtId, expiresAt))
{
return ValueTask.FromResult(true);
}
while (!cancellationToken.IsCancellationRequested)
{
if (!entries.TryGetValue(jwtId, out var existing))
{
if (entries.TryAdd(jwtId, expiresAt))
{
return ValueTask.FromResult(true);
}
continue;
}
if (existing > now)
{
return ValueTask.FromResult(false);
}
if (entries.TryUpdate(jwtId, expiresAt, existing))
{
return ValueTask.FromResult(true);
}
}
return ValueTask.FromResult(false);
}
private void RemoveExpired(DateTimeOffset now)
{
foreach (var entry in entries)
{
if (entry.Value <= now)
{
entries.TryRemove(entry.Key, out _);
}
}
}
}

View File

@@ -0,0 +1,53 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
namespace StellaOps.Scanner.Core.Security;
public sealed class RestartOnlyPluginGuard : IPluginCatalogGuard
{
private readonly ConcurrentDictionary<string, byte> plugins = new(StringComparer.OrdinalIgnoreCase);
private bool sealedState;
public RestartOnlyPluginGuard(IEnumerable<string>? initialPlugins = null)
{
if (initialPlugins is not null)
{
foreach (var plugin in initialPlugins)
{
var normalized = Normalize(plugin);
plugins.TryAdd(normalized, 0);
}
}
}
public IReadOnlyCollection<string> KnownPlugins => plugins.Keys.ToArray();
public bool IsSealed => Volatile.Read(ref sealedState);
public void EnsureRegistrationAllowed(string pluginPath)
{
ArgumentException.ThrowIfNullOrWhiteSpace(pluginPath);
var normalized = Normalize(pluginPath);
if (IsSealed && !plugins.ContainsKey(normalized))
{
throw new InvalidOperationException($"Plug-in '{pluginPath}' cannot be registered after startup. Restart required.");
}
plugins.TryAdd(normalized, 0);
}
public void Seal()
{
Volatile.Write(ref sealedState, true);
}
private static string Normalize(string path)
{
var full = Path.GetFullPath(path);
return full.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
}

View File

@@ -0,0 +1,66 @@
using System.Collections.ObjectModel;
using System.Linq;
namespace StellaOps.Scanner.Core.Security;
public readonly record struct ScannerOperationalToken(
string AccessToken,
string TokenType,
DateTimeOffset ExpiresAt,
IReadOnlyList<string> Scopes)
{
public bool IsExpired(TimeProvider timeProvider, TimeSpan refreshSkew)
{
ArgumentNullException.ThrowIfNull(timeProvider);
var now = timeProvider.GetUtcNow();
return now >= ExpiresAt - refreshSkew;
}
public static ScannerOperationalToken FromResult(
string accessToken,
string tokenType,
DateTimeOffset expiresAt,
IEnumerable<string> scopes)
{
ArgumentException.ThrowIfNullOrWhiteSpace(accessToken);
ArgumentException.ThrowIfNullOrWhiteSpace(tokenType);
IReadOnlyList<string> normalized = scopes switch
{
null => Array.Empty<string>(),
IReadOnlyList<string> readOnly => readOnly.Count == 0 ? Array.Empty<string>() : readOnly,
ICollection<string> collection => NormalizeCollection(collection),
_ => NormalizeEnumerable(scopes)
};
return new ScannerOperationalToken(
accessToken,
tokenType,
expiresAt,
normalized);
}
private static IReadOnlyList<string> NormalizeCollection(ICollection<string> collection)
{
if (collection.Count == 0)
{
return Array.Empty<string>();
}
if (collection is IReadOnlyList<string> readOnly)
{
return readOnly;
}
var buffer = new string[collection.Count];
collection.CopyTo(buffer, 0);
return new ReadOnlyCollection<string>(buffer);
}
private static IReadOnlyList<string> NormalizeEnumerable(IEnumerable<string> scopes)
{
var buffer = scopes.ToArray();
return buffer.Length == 0 ? Array.Empty<string>() : new ReadOnlyCollection<string>(buffer);
}
}

View File

@@ -0,0 +1,36 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
using StellaOps.Auth.Client;
namespace StellaOps.Scanner.Core.Security;
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddScannerAuthorityCore(
this IServiceCollection services,
Action<StellaOpsAuthClientOptions> configureAuthority,
Action<DpopValidationOptions>? configureDpop = null)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configureAuthority);
services.AddStellaOpsAuthClient(configureAuthority);
if (configureDpop is not null)
{
services.AddOptions<DpopValidationOptions>().Configure(configureDpop).PostConfigure(static options => options.Validate());
}
else
{
services.AddOptions<DpopValidationOptions>().PostConfigure(static options => options.Validate());
}
services.TryAddSingleton<IDpopReplayCache>(provider => new InMemoryDpopReplayCache(provider.GetService<TimeProvider>()));
services.TryAddSingleton<IDpopProofValidator, DpopProofValidator>();
services.TryAddSingleton<IAuthorityTokenSource, AuthorityTokenSource>();
services.TryAddSingleton<IPluginCatalogGuard, RestartOnlyPluginGuard>();
return services;
}
}

View File

@@ -0,0 +1,21 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Core.Serialization;
public static class ScannerJsonOptions
{
public static JsonSerializerOptions Default { get; } = CreateDefault();
public static JsonSerializerOptions CreateDefault(bool indent = false)
{
var options = new JsonSerializerOptions(JsonSerializerDefaults.Web)
{
WriteIndented = indent
};
options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase));
return options;
}
}

View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="8.0.1" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="8.0.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
# Scanner Core Task Board
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| SCANNER-CORE-09-501 | DONE (2025-10-18) | Scanner Core Guild | — | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3§4. | DTOs serialize deterministically, helpers produce reproducible IDs/timestamps, tests cover round-trips and hash derivation. |
| SCANNER-CORE-09-502 | DONE (2025-10-18) | Scanner Core Guild | SCANNER-CORE-09-501 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker. | Logging/metrics helpers allocate minimally, correlation IDs stable, ActivitySource emitted; tests assert determinism. |
| SCANNER-CORE-09-503 | DONE (2025-10-18) | Scanner Core Guild | SCANNER-CORE-09-501, SCANNER-CORE-09-502 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | Authority helpers cache tokens, DPoP validator rejects invalid proofs, plug-in guard prevents runtime additions; tests cover happy/error paths. |

View File

@@ -0,0 +1,136 @@
using System.Globalization;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Core.Utility;
public static class ScannerIdentifiers
{
private static readonly Guid ScanJobNamespace = new("d985aa76-8c2b-4cba-bac0-c98c90674f04");
private static readonly Guid CorrelationNamespace = new("7cde18f5-729e-4ea1-be3d-46fda4c55e38");
public static ScanJobId CreateJobId(
string imageReference,
string? imageDigest = null,
string? tenantId = null,
string? salt = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(imageReference);
var normalizedReference = NormalizeImageReference(imageReference);
var normalizedDigest = NormalizeDigest(imageDigest) ?? "none";
var normalizedTenant = string.IsNullOrWhiteSpace(tenantId) ? "global" : tenantId.Trim().ToLowerInvariant();
var normalizedSalt = (salt?.Trim() ?? string.Empty).ToLowerInvariant();
using var sha256 = SHA256.Create();
var payload = $"{normalizedReference}|{normalizedDigest}|{normalizedTenant}|{normalizedSalt}";
var hashed = sha256.ComputeHash(Encoding.UTF8.GetBytes(payload));
return new ScanJobId(CreateGuidFromHash(ScanJobNamespace, hashed));
}
public static string CreateCorrelationId(ScanJobId jobId, string? stage = null, string? suffix = null)
{
var normalizedStage = string.IsNullOrWhiteSpace(stage)
? "scan"
: stage.Trim().ToLowerInvariant().Replace(' ', '-');
var normalizedSuffix = string.IsNullOrWhiteSpace(suffix)
? string.Empty
: "-" + suffix.Trim().ToLowerInvariant().Replace(' ', '-');
return $"scan-{normalizedStage}-{jobId}{normalizedSuffix}";
}
public static string CreateDeterministicHash(params string[] segments)
{
if (segments is null || segments.Length == 0)
{
throw new ArgumentException("At least one segment must be provided.", nameof(segments));
}
using var sha256 = SHA256.Create();
var joined = string.Join('|', segments.Select(static s => s?.Trim() ?? string.Empty));
var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(joined));
return Convert.ToHexString(hash).ToLowerInvariant();
}
public static Guid CreateDeterministicGuid(Guid namespaceId, ReadOnlySpan<byte> nameBytes)
{
Span<byte> namespaceBytes = stackalloc byte[16];
namespaceId.TryWriteBytes(namespaceBytes);
Span<byte> buffer = stackalloc byte[namespaceBytes.Length + nameBytes.Length];
namespaceBytes.CopyTo(buffer);
nameBytes.CopyTo(buffer[namespaceBytes.Length..]);
Span<byte> hash = stackalloc byte[32];
SHA256.TryHashData(buffer, hash, out _);
Span<byte> guidBytes = stackalloc byte[16];
hash[..16].CopyTo(guidBytes);
guidBytes[6] = (byte)((guidBytes[6] & 0x0F) | 0x50);
guidBytes[8] = (byte)((guidBytes[8] & 0x3F) | 0x80);
return new Guid(guidBytes);
}
public static string NormalizeImageReference(string reference)
{
ArgumentException.ThrowIfNullOrWhiteSpace(reference);
var trimmed = reference.Trim();
var atIndex = trimmed.IndexOf('@');
if (atIndex > 0)
{
var prefix = trimmed[..atIndex].ToLowerInvariant();
return $"{prefix}{trimmed[atIndex..]}";
}
var colonIndex = trimmed.IndexOf(':');
if (colonIndex > 0)
{
var name = trimmed[..colonIndex].ToLowerInvariant();
var tag = trimmed[(colonIndex + 1)..];
return $"{name}:{tag}";
}
return trimmed.ToLowerInvariant();
}
public static string? NormalizeDigest(string? digest)
{
if (string.IsNullOrWhiteSpace(digest))
{
return null;
}
var trimmed = digest.Trim();
var parts = trimmed.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
if (parts.Length != 2)
{
return trimmed.ToLowerInvariant();
}
return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}";
}
public static string CreateDeterministicCorrelation(string audience, ScanJobId jobId, string? component = null)
{
using var sha256 = SHA256.Create();
var payload = $"{audience.Trim().ToLowerInvariant()}|{jobId}|{component?.Trim().ToLowerInvariant() ?? string.Empty}";
var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(payload));
var guid = CreateGuidFromHash(CorrelationNamespace, hash);
return $"corr-{guid.ToString("n", CultureInfo.InvariantCulture)}";
}
private static Guid CreateGuidFromHash(Guid namespaceId, ReadOnlySpan<byte> hash)
{
Span<byte> guidBytes = stackalloc byte[16];
hash[..16].CopyTo(guidBytes);
guidBytes[6] = (byte)((guidBytes[6] & 0x0F) | 0x50);
guidBytes[8] = (byte)((guidBytes[8] & 0x3F) | 0x80);
return new Guid(guidBytes);
}
}

View File

@@ -0,0 +1,43 @@
using System.Globalization;
namespace StellaOps.Scanner.Core.Utility;
public static class ScannerTimestamps
{
private const long TicksPerMicrosecond = TimeSpan.TicksPerMillisecond / 1000;
public static DateTimeOffset Normalize(DateTimeOffset value)
{
var utc = value.ToUniversalTime();
var ticks = utc.Ticks - (utc.Ticks % TicksPerMicrosecond);
return new DateTimeOffset(ticks, TimeSpan.Zero);
}
public static DateTimeOffset UtcNow(TimeProvider? provider = null)
=> Normalize((provider ?? TimeProvider.System).GetUtcNow());
public static string ToIso8601(DateTimeOffset value)
=> Normalize(value).ToString("yyyy-MM-dd'T'HH:mm:ss.ffffff'Z'", CultureInfo.InvariantCulture);
public static bool TryParseIso8601(string? value, out DateTimeOffset timestamp)
{
if (string.IsNullOrWhiteSpace(value))
{
timestamp = default;
return false;
}
if (DateTimeOffset.TryParse(
value,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal,
out var parsed))
{
timestamp = Normalize(parsed);
return true;
}
timestamp = default;
return false;
}
}

View File

@@ -0,0 +1,353 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Scanner.Queue;
using Xunit;
namespace StellaOps.Scanner.Queue.Tests;
public sealed class QueueLeaseIntegrationTests
{
private readonly ScannerQueueOptions _options = new()
{
MaxDeliveryAttempts = 3,
RetryInitialBackoff = TimeSpan.FromMilliseconds(1),
RetryMaxBackoff = TimeSpan.FromMilliseconds(5),
DefaultLeaseDuration = TimeSpan.FromSeconds(5)
};
[Fact]
public async Task Enqueue_ShouldDeduplicate_ByIdempotencyKey()
{
var clock = new FakeTimeProvider();
var queue = new InMemoryScanQueue(_options, clock);
var payload = new byte[] { 1, 2, 3 };
var message = new ScanQueueMessage("job-1", payload)
{
IdempotencyKey = "idem-1"
};
var first = await queue.EnqueueAsync(message);
first.Deduplicated.Should().BeFalse();
var second = await queue.EnqueueAsync(message);
second.Deduplicated.Should().BeTrue();
}
[Fact]
public async Task Lease_Acknowledge_ShouldRemoveFromQueue()
{
var clock = new FakeTimeProvider();
var queue = new InMemoryScanQueue(_options, clock);
var message = new ScanQueueMessage("job-ack", new byte[] { 42 });
await queue.EnqueueAsync(message);
var lease = await LeaseSingleAsync(queue, consumer: "worker-1");
lease.Should().NotBeNull();
await lease!.AcknowledgeAsync();
var afterAck = await queue.LeaseAsync(new QueueLeaseRequest("worker-1", 1, TimeSpan.FromSeconds(1)));
afterAck.Should().BeEmpty();
}
[Fact]
public async Task Release_WithRetry_ShouldDeadLetterAfterMaxAttempts()
{
var clock = new FakeTimeProvider();
var queue = new InMemoryScanQueue(_options, clock);
var message = new ScanQueueMessage("job-retry", new byte[] { 5 });
await queue.EnqueueAsync(message);
for (var attempt = 1; attempt <= _options.MaxDeliveryAttempts; attempt++)
{
var lease = await LeaseSingleAsync(queue, consumer: $"worker-{attempt}");
lease.Should().NotBeNull();
await lease!.ReleaseAsync(QueueReleaseDisposition.Retry);
}
queue.DeadLetters.Should().ContainSingle(dead => dead.JobId == "job-retry");
}
[Fact]
public async Task Retry_ShouldIncreaseAttemptOnNextLease()
{
var clock = new FakeTimeProvider();
var queue = new InMemoryScanQueue(_options, clock);
await queue.EnqueueAsync(new ScanQueueMessage("job-retry-attempt", new byte[] { 77 }));
var firstLease = await LeaseSingleAsync(queue, "worker-retry");
firstLease.Should().NotBeNull();
firstLease!.Attempt.Should().Be(1);
await firstLease.ReleaseAsync(QueueReleaseDisposition.Retry);
var secondLease = await LeaseSingleAsync(queue, "worker-retry");
secondLease.Should().NotBeNull();
secondLease!.Attempt.Should().Be(2);
}
private static async Task<IScanQueueLease?> LeaseSingleAsync(InMemoryScanQueue queue, string consumer)
{
var leases = await queue.LeaseAsync(new QueueLeaseRequest(consumer, 1, TimeSpan.FromSeconds(1)));
return leases.FirstOrDefault();
}
private sealed class InMemoryScanQueue : IScanQueue
{
private readonly ScannerQueueOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ConcurrentQueue<QueueEntry> _ready = new();
private readonly ConcurrentDictionary<string, QueueEntry> _idempotency = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, QueueEntry> _inFlight = new(StringComparer.Ordinal);
private readonly List<QueueEntry> _deadLetters = new();
private long _sequence;
public InMemoryScanQueue(ScannerQueueOptions options, TimeProvider timeProvider)
{
_options = options;
_timeProvider = timeProvider;
}
public IReadOnlyList<QueueEntry> DeadLetters => _deadLetters;
public ValueTask<QueueEnqueueResult> EnqueueAsync(ScanQueueMessage message, CancellationToken cancellationToken = default)
{
var token = message.IdempotencyKey ?? message.JobId;
if (_idempotency.TryGetValue(token, out var existing))
{
return ValueTask.FromResult(new QueueEnqueueResult(existing.SequenceId, true));
}
var entry = new QueueEntry(
sequenceId: Interlocked.Increment(ref _sequence).ToString(),
jobId: message.JobId,
payload: message.Payload.ToArray(),
idempotencyKey: token,
attempt: 1,
enqueuedAt: _timeProvider.GetUtcNow());
_idempotency[token] = entry;
_ready.Enqueue(entry);
return ValueTask.FromResult(new QueueEnqueueResult(entry.SequenceId, false));
}
public ValueTask<IReadOnlyList<IScanQueueLease>> LeaseAsync(QueueLeaseRequest request, CancellationToken cancellationToken = default)
{
var now = _timeProvider.GetUtcNow();
var leases = new List<IScanQueueLease>(request.BatchSize);
while (leases.Count < request.BatchSize && _ready.TryDequeue(out var entry))
{
entry.Attempt = Math.Max(entry.Attempt, entry.Deliveries + 1);
entry.Deliveries = entry.Attempt;
entry.LastLeaseAt = now;
_inFlight[entry.SequenceId] = entry;
var lease = new InMemoryLease(
this,
entry,
request.Consumer,
now,
request.LeaseDuration);
leases.Add(lease);
}
return ValueTask.FromResult<IReadOnlyList<IScanQueueLease>>(leases);
}
public ValueTask<IReadOnlyList<IScanQueueLease>> ClaimExpiredLeasesAsync(QueueClaimOptions options, CancellationToken cancellationToken = default)
{
var now = _timeProvider.GetUtcNow();
var leases = _inFlight.Values
.Where(entry => now - entry.LastLeaseAt >= options.MinIdleTime)
.Take(options.BatchSize)
.Select(entry => new InMemoryLease(this, entry, options.ClaimantConsumer, now, _options.DefaultLeaseDuration))
.Cast<IScanQueueLease>()
.ToList();
return ValueTask.FromResult<IReadOnlyList<IScanQueueLease>>(leases);
}
internal Task AcknowledgeAsync(QueueEntry entry)
{
_inFlight.TryRemove(entry.SequenceId, out _);
_idempotency.TryRemove(entry.IdempotencyKey, out _);
return Task.CompletedTask;
}
internal Task<DateTimeOffset> RenewAsync(QueueEntry entry, TimeSpan leaseDuration)
{
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
entry.LeaseExpiresAt = expires;
return Task.FromResult(expires);
}
internal Task ReleaseAsync(QueueEntry entry, QueueReleaseDisposition disposition)
{
if (disposition == QueueReleaseDisposition.Retry && entry.Attempt >= _options.MaxDeliveryAttempts)
{
return DeadLetterAsync(entry, $"max-delivery-attempts:{entry.Attempt}");
}
if (disposition == QueueReleaseDisposition.Retry)
{
entry.Attempt++;
_ready.Enqueue(entry);
}
else
{
_idempotency.TryRemove(entry.IdempotencyKey, out _);
}
_inFlight.TryRemove(entry.SequenceId, out _);
return Task.CompletedTask;
}
internal Task DeadLetterAsync(QueueEntry entry, string reason)
{
entry.DeadLetterReason = reason;
_inFlight.TryRemove(entry.SequenceId, out _);
_idempotency.TryRemove(entry.IdempotencyKey, out _);
_deadLetters.Add(entry);
return Task.CompletedTask;
}
private sealed class InMemoryLease : IScanQueueLease
{
private readonly InMemoryScanQueue _owner;
private readonly QueueEntry _entry;
private int _completed;
public InMemoryLease(
InMemoryScanQueue owner,
QueueEntry entry,
string consumer,
DateTimeOffset now,
TimeSpan leaseDuration)
{
_owner = owner;
_entry = entry;
Consumer = consumer;
MessageId = entry.SequenceId;
JobId = entry.JobId;
Payload = entry.Payload;
Attempt = entry.Attempt;
EnqueuedAt = entry.EnqueuedAt;
LeaseExpiresAt = now.Add(leaseDuration);
IdempotencyKey = entry.IdempotencyKey;
Attributes = entry.Attributes;
}
public string MessageId { get; }
public string JobId { get; }
public ReadOnlyMemory<byte> Payload { get; }
public int Attempt { get; }
public DateTimeOffset EnqueuedAt { get; }
public DateTimeOffset LeaseExpiresAt { get; private set; }
public string Consumer { get; }
public string? IdempotencyKey { get; }
public IReadOnlyDictionary<string, string> Attributes { get; }
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
{
if (TryComplete())
{
return _owner.AcknowledgeAsync(_entry);
}
return Task.CompletedTask;
}
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
{
return RenewInternalAsync(leaseDuration);
}
public Task ReleaseAsync(QueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
{
if (TryComplete())
{
return _owner.ReleaseAsync(_entry, disposition);
}
return Task.CompletedTask;
}
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
{
if (TryComplete())
{
return _owner.DeadLetterAsync(_entry, reason);
}
return Task.CompletedTask;
}
private async Task RenewInternalAsync(TimeSpan leaseDuration)
{
var expires = await _owner.RenewAsync(_entry, leaseDuration).ConfigureAwait(false);
LeaseExpiresAt = expires;
}
private bool TryComplete()
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
}
internal sealed class QueueEntry
{
public QueueEntry(string sequenceId, string jobId, byte[] payload, string idempotencyKey, int attempt, DateTimeOffset enqueuedAt)
{
SequenceId = sequenceId;
JobId = jobId;
Payload = payload;
IdempotencyKey = idempotencyKey;
Attempt = attempt;
EnqueuedAt = enqueuedAt;
LastLeaseAt = enqueuedAt;
Attributes = new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(StringComparer.Ordinal));
}
public string SequenceId { get; }
public string JobId { get; }
public byte[] Payload { get; }
public string IdempotencyKey { get; }
public int Attempt { get; set; }
public int Deliveries { get; set; }
public DateTimeOffset EnqueuedAt { get; }
public DateTimeOffset LeaseExpiresAt { get; set; }
public DateTimeOffset LastLeaseAt { get; set; }
public IReadOnlyDictionary<string, string> Attributes { get; }
public string? DeadLetterReason { get; set; }
}
}
}

View File

@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,15 @@
# StellaOps.Scanner.Queue — Agent Charter
## Mission
Deliver the scanner job queue backbone defined in `docs/ARCHITECTURE_SCANNER.md`, providing deterministic, offline-friendly leasing semantics for WebService producers and Worker consumers.
## Responsibilities
- Define queue abstractions with idempotent enqueue tokens, acknowledgement, lease renewal, and claim support.
- Ship first-party adapters for Redis Streams and NATS JetStream, respecting offline deployments and allow-listed hosts.
- Surface health probes, structured diagnostics, and metrics needed by Scanner WebService/Worker.
- Document operational expectations and configuration binding hooks.
## Interfaces & Dependencies
- Consumes shared configuration primitives from `StellaOps.Configuration`.
- Exposes dependency injection extensions for `StellaOps.DependencyInjection`.
- Targets `net10.0` (preview) and aligns with scanner DTOs once `StellaOps.Scanner.Core` lands.

View File

@@ -0,0 +1,20 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Queue;
public interface IScanQueue
{
ValueTask<QueueEnqueueResult> EnqueueAsync(
ScanQueueMessage message,
CancellationToken cancellationToken = default);
ValueTask<IReadOnlyList<IScanQueueLease>> LeaseAsync(
QueueLeaseRequest request,
CancellationToken cancellationToken = default);
ValueTask<IReadOnlyList<IScanQueueLease>> ClaimExpiredLeasesAsync(
QueueClaimOptions options,
CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,35 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Queue;
public interface IScanQueueLease
{
string MessageId { get; }
string JobId { get; }
ReadOnlyMemory<byte> Payload { get; }
int Attempt { get; }
DateTimeOffset EnqueuedAt { get; }
DateTimeOffset LeaseExpiresAt { get; }
string Consumer { get; }
string? IdempotencyKey { get; }
IReadOnlyDictionary<string, string> Attributes { get; }
Task AcknowledgeAsync(CancellationToken cancellationToken = default);
Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default);
Task ReleaseAsync(QueueReleaseDisposition disposition, CancellationToken cancellationToken = default);
Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,644 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using NATS.Client.Core;
using NATS.Client.JetStream;
using NATS.Client.JetStream.Models;
namespace StellaOps.Scanner.Queue.Nats;
internal sealed class NatsScanQueue : IScanQueue, IAsyncDisposable
{
private const string TransportName = "nats";
private static readonly INatsSerializer<byte[]> PayloadSerializer = NatsRawSerializer<byte[]>.Default;
private readonly ScannerQueueOptions _queueOptions;
private readonly NatsQueueOptions _options;
private readonly ILogger<NatsScanQueue> _logger;
private readonly TimeProvider _timeProvider;
private readonly SemaphoreSlim _connectionGate = new(1, 1);
private readonly Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>> _connectionFactory;
private NatsConnection? _connection;
private NatsJSContext? _jsContext;
private INatsJSConsumer? _consumer;
private bool _disposed;
public NatsScanQueue(
ScannerQueueOptions queueOptions,
NatsQueueOptions options,
ILogger<NatsScanQueue> logger,
TimeProvider timeProvider,
Func<NatsOpts, CancellationToken, ValueTask<NatsConnection>>? connectionFactory = null)
{
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_connectionFactory = connectionFactory ?? ((opts, cancellationToken) => new ValueTask<NatsConnection>(new NatsConnection(opts)));
if (string.IsNullOrWhiteSpace(_options.Url))
{
throw new InvalidOperationException("NATS connection URL must be configured for the scanner queue.");
}
}
public async ValueTask<QueueEnqueueResult> EnqueueAsync(
ScanQueueMessage message,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(message);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
var idempotencyKey = message.IdempotencyKey ?? message.JobId;
var headers = BuildHeaders(message, idempotencyKey);
var publishOpts = new NatsJSPubOpts
{
MsgId = idempotencyKey,
RetryAttempts = 0
};
var ack = await js.PublishAsync(
_options.Subject,
message.Payload.ToArray(),
PayloadSerializer,
publishOpts,
headers,
cancellationToken)
.ConfigureAwait(false);
if (ack.Duplicate)
{
_logger.LogDebug(
"Duplicate NATS enqueue detected for job {JobId} (token {Token}).",
message.JobId,
idempotencyKey);
QueueMetrics.RecordDeduplicated(TransportName);
return new QueueEnqueueResult(ack.Seq.ToString(), true);
}
QueueMetrics.RecordEnqueued(TransportName);
_logger.LogDebug(
"Enqueued job {JobId} into NATS stream {Stream} with sequence {Sequence}.",
message.JobId,
ack.Stream,
ack.Seq);
return new QueueEnqueueResult(ack.Seq.ToString(), false);
}
public async ValueTask<IReadOnlyList<IScanQueueLease>> LeaseAsync(
QueueLeaseRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
var fetchOpts = new NatsJSFetchOpts
{
MaxMsgs = request.BatchSize,
Expires = request.LeaseDuration,
IdleHeartbeat = _options.IdleHeartbeat
};
var now = _timeProvider.GetUtcNow();
var leases = new List<IScanQueueLease>(capacity: request.BatchSize);
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
{
var lease = CreateLease(msg, request.Consumer, now, request.LeaseDuration);
if (lease is not null)
{
leases.Add(lease);
}
}
return leases;
}
public async ValueTask<IReadOnlyList<IScanQueueLease>> ClaimExpiredLeasesAsync(
QueueClaimOptions options,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false);
var fetchOpts = new NatsJSFetchOpts
{
MaxMsgs = options.BatchSize,
Expires = options.MinIdleTime,
IdleHeartbeat = _options.IdleHeartbeat
};
var now = _timeProvider.GetUtcNow();
var leases = new List<IScanQueueLease>(options.BatchSize);
await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false))
{
var deliveries = (int)(msg.Metadata?.NumDelivered ?? 1);
if (deliveries <= 1)
{
// Fresh message; surface back to queue and continue.
await msg.NakAsync(new AckOpts(), TimeSpan.Zero, cancellationToken).ConfigureAwait(false);
continue;
}
var lease = CreateLease(msg, options.ClaimantConsumer, now, _queueOptions.DefaultLeaseDuration);
if (lease is not null)
{
leases.Add(lease);
}
}
return leases;
}
public async ValueTask DisposeAsync()
{
if (_disposed)
{
return;
}
_disposed = true;
if (_connection is not null)
{
await _connection.DisposeAsync().ConfigureAwait(false);
}
_connectionGate.Dispose();
GC.SuppressFinalize(this);
}
internal async Task AcknowledgeAsync(
NatsScanQueueLease lease,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
await lease.Message.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
QueueMetrics.RecordAck(TransportName);
_logger.LogDebug(
"Acknowledged job {JobId} (seq {Seq}).",
lease.JobId,
lease.MessageId);
}
internal async Task RenewLeaseAsync(
NatsScanQueueLease lease,
TimeSpan leaseDuration,
CancellationToken cancellationToken)
{
await lease.Message.AckProgressAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
lease.RefreshLease(expires);
_logger.LogDebug(
"Renewed NATS lease for job {JobId} until {Expires:u}.",
lease.JobId,
expires);
}
internal async Task ReleaseAsync(
NatsScanQueueLease lease,
QueueReleaseDisposition disposition,
CancellationToken cancellationToken)
{
if (disposition == QueueReleaseDisposition.Retry
&& lease.Attempt >= _queueOptions.MaxDeliveryAttempts)
{
_logger.LogWarning(
"Job {JobId} reached max delivery attempts ({Attempts}); shipping to dead-letter stream.",
lease.JobId,
lease.Attempt);
await DeadLetterAsync(
lease,
$"max-delivery-attempts:{lease.Attempt}",
cancellationToken).ConfigureAwait(false);
return;
}
if (!lease.TryBeginCompletion())
{
return;
}
if (disposition == QueueReleaseDisposition.Retry)
{
QueueMetrics.RecordRetry(TransportName);
var delay = CalculateBackoff(lease.Attempt);
await lease.Message.NakAsync(new AckOpts(), delay, cancellationToken).ConfigureAwait(false);
_logger.LogWarning(
"Rescheduled job {JobId} via NATS NAK with delay {Delay} (attempt {Attempt}).",
lease.JobId,
delay,
lease.Attempt);
}
else
{
await lease.Message.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
QueueMetrics.RecordAck(TransportName);
_logger.LogInformation(
"Abandoned job {JobId} after {Attempt} attempt(s).",
lease.JobId,
lease.Attempt);
}
}
internal async Task DeadLetterAsync(
NatsScanQueueLease lease,
string reason,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
await lease.Message.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false);
var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
var headers = BuildDeadLetterHeaders(lease, reason);
await js.PublishAsync(
_options.DeadLetterSubject,
lease.Payload.ToArray(),
PayloadSerializer,
new NatsJSPubOpts(),
headers,
cancellationToken)
.ConfigureAwait(false);
QueueMetrics.RecordDeadLetter(TransportName);
_logger.LogError(
"Dead-lettered job {JobId} (attempt {Attempt}): {Reason}",
lease.JobId,
lease.Attempt,
reason);
}
private async Task<NatsJSContext> GetJetStreamAsync(CancellationToken cancellationToken)
{
if (_jsContext is not null)
{
return _jsContext;
}
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
_jsContext ??= new NatsJSContext(connection);
return _jsContext;
}
finally
{
_connectionGate.Release();
}
}
private async ValueTask<INatsJSConsumer> EnsureStreamAndConsumerAsync(
NatsJSContext js,
CancellationToken cancellationToken)
{
if (_consumer is not null)
{
return _consumer;
}
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_consumer is not null)
{
return _consumer;
}
await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false);
await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false);
var consumerConfig = new ConsumerConfig
{
DurableName = _options.DurableConsumer,
AckPolicy = ConsumerConfigAckPolicy.Explicit,
ReplayPolicy = ConsumerConfigReplayPolicy.Instant,
DeliverPolicy = ConsumerConfigDeliverPolicy.All,
AckWait = ToNanoseconds(_options.AckWait),
MaxAckPending = _options.MaxInFlight,
MaxDeliver = Math.Max(1, _queueOptions.MaxDeliveryAttempts),
FilterSubjects = new[] { _options.Subject }
};
try
{
_consumer = await js.CreateConsumerAsync(
_options.Stream,
consumerConfig,
cancellationToken)
.ConfigureAwait(false);
}
catch (NatsJSApiException apiEx)
{
_logger.LogDebug(apiEx,
"CreateConsumerAsync failed with code {Code}; attempting to fetch existing durable consumer {Durable}.",
apiEx.Error?.Code,
_options.DurableConsumer);
_consumer = await js.GetConsumerAsync(
_options.Stream,
_options.DurableConsumer,
cancellationToken)
.ConfigureAwait(false);
}
return _consumer;
}
finally
{
_connectionGate.Release();
}
}
private async Task<NatsConnection> EnsureConnectionAsync(CancellationToken cancellationToken)
{
if (_connection is not null)
{
return _connection;
}
await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_connection is not null)
{
return _connection;
}
var opts = new NatsOpts
{
Url = _options.Url!,
Name = "stellaops-scanner-queue",
CommandTimeout = TimeSpan.FromSeconds(10),
RequestTimeout = TimeSpan.FromSeconds(20),
PingInterval = TimeSpan.FromSeconds(30)
};
_connection = await _connectionFactory(opts, cancellationToken).ConfigureAwait(false);
await _connection.ConnectAsync().ConfigureAwait(false);
return _connection;
}
finally
{
_connectionGate.Release();
}
}
private async Task EnsureStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
{
try
{
await js.GetStreamAsync(
_options.Stream,
new StreamInfoRequest(),
cancellationToken)
.ConfigureAwait(false);
}
catch (NatsJSApiException)
{
var config = new StreamConfig(
name: _options.Stream,
subjects: new[] { _options.Subject })
{
Retention = StreamConfigRetention.Workqueue,
Storage = StreamConfigStorage.File,
MaxConsumers = -1,
MaxMsgs = -1,
MaxBytes = -1,
MaxAge = 0
};
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Created NATS JetStream stream {Stream} ({Subject}).", _options.Stream, _options.Subject);
}
}
private async Task EnsureDeadLetterStreamAsync(NatsJSContext js, CancellationToken cancellationToken)
{
try
{
await js.GetStreamAsync(
_options.DeadLetterStream,
new StreamInfoRequest(),
cancellationToken)
.ConfigureAwait(false);
}
catch (NatsJSApiException)
{
var config = new StreamConfig(
name: _options.DeadLetterStream,
subjects: new[] { _options.DeadLetterSubject })
{
Retention = StreamConfigRetention.Workqueue,
Storage = StreamConfigStorage.File,
MaxConsumers = -1,
MaxMsgs = -1,
MaxBytes = -1,
MaxAge = ToNanoseconds(_queueOptions.DeadLetter.Retention)
};
await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Created NATS dead-letter stream {Stream} ({Subject}).", _options.DeadLetterStream, _options.DeadLetterSubject);
}
}
internal async ValueTask PingAsync(CancellationToken cancellationToken)
{
var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false);
await connection.PingAsync(cancellationToken).ConfigureAwait(false);
}
private NatsScanQueueLease? CreateLease(
NatsJSMsg<byte[]> message,
string consumer,
DateTimeOffset now,
TimeSpan leaseDuration)
{
var headers = message.Headers;
if (headers is null)
{
return null;
}
if (!headers.TryGetValue(QueueEnvelopeFields.JobId, out var jobIdValues) || jobIdValues.Count == 0)
{
return null;
}
var jobId = jobIdValues[0]!;
var idempotencyKey = headers.TryGetValue(QueueEnvelopeFields.IdempotencyKey, out var idemValues) && idemValues.Count > 0
? idemValues[0]
: null;
var enqueuedAt = headers.TryGetValue(QueueEnvelopeFields.EnqueuedAt, out var enqueuedValues) && enqueuedValues.Count > 0
&& long.TryParse(enqueuedValues[0], out var unix)
? DateTimeOffset.FromUnixTimeMilliseconds(unix)
: now;
var attempt = headers.TryGetValue(QueueEnvelopeFields.Attempt, out var attemptValues) && attemptValues.Count > 0
&& int.TryParse(attemptValues[0], out var parsedAttempt)
? parsedAttempt
: 1;
if (message.Metadata?.NumDelivered is ulong delivered && delivered > 0)
{
var deliveredInt = delivered > int.MaxValue ? int.MaxValue : (int)delivered;
if (deliveredInt > attempt)
{
attempt = deliveredInt;
}
}
var leaseExpires = now.Add(leaseDuration);
var attributes = ExtractAttributes(headers);
var messageId = message.Metadata?.Sequence.Stream.ToString() ?? Guid.NewGuid().ToString("n");
return new NatsScanQueueLease(
this,
message,
messageId,
jobId,
message.Data ?? Array.Empty<byte>(),
attempt,
enqueuedAt,
leaseExpires,
consumer,
idempotencyKey,
attributes);
}
private static IReadOnlyDictionary<string, string> ExtractAttributes(NatsHeaders headers)
{
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var key in headers.Keys)
{
if (!key.StartsWith(QueueEnvelopeFields.AttributePrefix, StringComparison.Ordinal))
{
continue;
}
if (headers.TryGetValue(key, out var values) && values.Count > 0)
{
attributes[key[QueueEnvelopeFields.AttributePrefix.Length..]] = values[0]!;
}
}
return attributes.Count == 0
? EmptyReadOnlyDictionary<string, string>.Instance
: new ReadOnlyDictionary<string, string>(attributes);
}
private NatsHeaders BuildHeaders(ScanQueueMessage message, string idempotencyKey)
{
var headers = new NatsHeaders
{
{ QueueEnvelopeFields.JobId, message.JobId },
{ QueueEnvelopeFields.IdempotencyKey, idempotencyKey },
{ QueueEnvelopeFields.Attempt, "1" },
{ QueueEnvelopeFields.EnqueuedAt, _timeProvider.GetUtcNow().ToUnixTimeMilliseconds().ToString() }
};
if (!string.IsNullOrEmpty(message.TraceId))
{
headers.Add(QueueEnvelopeFields.TraceId, message.TraceId!);
}
if (message.Attributes is not null)
{
foreach (var kvp in message.Attributes)
{
headers.Add(QueueEnvelopeFields.AttributePrefix + kvp.Key, kvp.Value);
}
}
return headers;
}
private NatsHeaders BuildDeadLetterHeaders(NatsScanQueueLease lease, string reason)
{
var headers = new NatsHeaders
{
{ QueueEnvelopeFields.JobId, lease.JobId },
{ QueueEnvelopeFields.IdempotencyKey, lease.IdempotencyKey ?? lease.JobId },
{ QueueEnvelopeFields.Attempt, lease.Attempt.ToString() },
{ QueueEnvelopeFields.EnqueuedAt, lease.EnqueuedAt.ToUnixTimeMilliseconds().ToString() },
{ "deadletter-reason", reason }
};
foreach (var kvp in lease.Attributes)
{
headers.Add(QueueEnvelopeFields.AttributePrefix + kvp.Key, kvp.Value);
}
return headers;
}
private TimeSpan CalculateBackoff(int attempt)
{
var configuredInitial = _options.RetryDelay > TimeSpan.Zero
? _options.RetryDelay
: _queueOptions.RetryInitialBackoff;
if (configuredInitial <= TimeSpan.Zero)
{
return TimeSpan.Zero;
}
if (attempt <= 1)
{
return configuredInitial;
}
var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero
? _queueOptions.RetryMaxBackoff
: configuredInitial;
var exponent = attempt - 1;
var scaledTicks = configuredInitial.Ticks * Math.Pow(2, exponent - 1);
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
var resultTicks = Math.Max(configuredInitial.Ticks, (long)cappedTicks);
return TimeSpan.FromTicks(resultTicks);
}
private static long ToNanoseconds(TimeSpan timeSpan)
=> timeSpan <= TimeSpan.Zero ? 0 : timeSpan.Ticks * 100L;
private static class EmptyReadOnlyDictionary<TKey, TValue>
where TKey : notnull
{
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
}
}

View File

@@ -0,0 +1,78 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using NATS.Client.JetStream;
namespace StellaOps.Scanner.Queue.Nats;
internal sealed class NatsScanQueueLease : IScanQueueLease
{
private readonly NatsScanQueue _queue;
private readonly NatsJSMsg<byte[]> _message;
private int _completed;
internal NatsScanQueueLease(
NatsScanQueue queue,
NatsJSMsg<byte[]> message,
string messageId,
string jobId,
byte[] payload,
int attempt,
DateTimeOffset enqueuedAt,
DateTimeOffset leaseExpiresAt,
string consumer,
string? idempotencyKey,
IReadOnlyDictionary<string, string> attributes)
{
_queue = queue;
_message = message;
MessageId = messageId;
JobId = jobId;
Payload = payload;
Attempt = attempt;
EnqueuedAt = enqueuedAt;
LeaseExpiresAt = leaseExpiresAt;
Consumer = consumer;
IdempotencyKey = idempotencyKey;
Attributes = attributes;
}
public string MessageId { get; }
public string JobId { get; }
public ReadOnlyMemory<byte> Payload { get; }
public int Attempt { get; internal set; }
public DateTimeOffset EnqueuedAt { get; }
public DateTimeOffset LeaseExpiresAt { get; private set; }
public string Consumer { get; }
public string? IdempotencyKey { get; }
public IReadOnlyDictionary<string, string> Attributes { get; }
internal NatsJSMsg<byte[]> Message => _message;
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
=> _queue.AcknowledgeAsync(this, cancellationToken);
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
public Task ReleaseAsync(QueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
internal bool TryBeginCompletion()
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
internal void RefreshLease(DateTimeOffset expiresAt)
=> LeaseExpiresAt = expiresAt;
}

View File

@@ -0,0 +1,12 @@
namespace StellaOps.Scanner.Queue;
internal static class QueueEnvelopeFields
{
public const string Payload = "payload";
public const string JobId = "jobId";
public const string IdempotencyKey = "idempotency";
public const string Attempt = "attempt";
public const string EnqueuedAt = "enqueuedAt";
public const string TraceId = "traceId";
public const string AttributePrefix = "attr:";
}

View File

@@ -0,0 +1,28 @@
using System.Diagnostics.Metrics;
namespace StellaOps.Scanner.Queue;
internal static class QueueMetrics
{
private const string TransportTagName = "transport";
private static readonly Meter Meter = new("StellaOps.Scanner.Queue");
private static readonly Counter<long> EnqueuedCounter = Meter.CreateCounter<long>("scanner_queue_enqueued_total");
private static readonly Counter<long> DeduplicatedCounter = Meter.CreateCounter<long>("scanner_queue_deduplicated_total");
private static readonly Counter<long> AckCounter = Meter.CreateCounter<long>("scanner_queue_ack_total");
private static readonly Counter<long> RetryCounter = Meter.CreateCounter<long>("scanner_queue_retry_total");
private static readonly Counter<long> DeadLetterCounter = Meter.CreateCounter<long>("scanner_queue_deadletter_total");
public static void RecordEnqueued(string transport) => EnqueuedCounter.Add(1, BuildTags(transport));
public static void RecordDeduplicated(string transport) => DeduplicatedCounter.Add(1, BuildTags(transport));
public static void RecordAck(string transport) => AckCounter.Add(1, BuildTags(transport));
public static void RecordRetry(string transport) => RetryCounter.Add(1, BuildTags(transport));
public static void RecordDeadLetter(string transport) => DeadLetterCounter.Add(1, BuildTags(transport));
private static KeyValuePair<string, object?>[] BuildTags(string transport)
=> new[] { new KeyValuePair<string, object?>(TransportTagName, transport) };
}

View File

@@ -0,0 +1,7 @@
namespace StellaOps.Scanner.Queue;
public enum QueueTransportKind
{
Redis,
Nats
}

View File

@@ -0,0 +1,764 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StackExchange.Redis;
namespace StellaOps.Scanner.Queue.Redis;
internal sealed class RedisScanQueue : IScanQueue, IAsyncDisposable
{
private const string TransportName = "redis";
private readonly ScannerQueueOptions _queueOptions;
private readonly RedisQueueOptions _options;
private readonly ILogger<RedisScanQueue> _logger;
private readonly TimeProvider _timeProvider;
private readonly SemaphoreSlim _connectionLock = new(1, 1);
private readonly SemaphoreSlim _groupInitLock = new(1, 1);
private readonly Func<ConfigurationOptions, Task<IConnectionMultiplexer>> _connectionFactory;
private IConnectionMultiplexer? _connection;
private volatile bool _groupInitialized;
private bool _disposed;
private string BuildIdempotencyKey(string key)
=> string.Concat(_options.IdempotencyKeyPrefix, key);
public RedisScanQueue(
ScannerQueueOptions queueOptions,
RedisQueueOptions options,
ILogger<RedisScanQueue> logger,
TimeProvider timeProvider,
Func<ConfigurationOptions, Task<IConnectionMultiplexer>>? connectionFactory = null)
{
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_connectionFactory = connectionFactory ?? (config => Task.FromResult<IConnectionMultiplexer>(ConnectionMultiplexer.Connect(config)));
if (string.IsNullOrWhiteSpace(_options.ConnectionString))
{
throw new InvalidOperationException("Redis connection string must be configured for the scanner queue.");
}
}
public async ValueTask<QueueEnqueueResult> EnqueueAsync(
ScanQueueMessage message,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(message);
cancellationToken.ThrowIfCancellationRequested();
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
var attempt = 1;
var entries = BuildEntries(message, now, attempt);
var messageId = await AddToStreamAsync(
db,
_options.StreamName,
entries,
_options.ApproximateMaxLength,
_options.ApproximateMaxLength is not null)
.ConfigureAwait(false);
var idempotencyToken = message.IdempotencyKey ?? message.JobId;
var idempotencyKey = BuildIdempotencyKey(idempotencyToken);
var stored = await db.StringSetAsync(
key: idempotencyKey,
value: messageId,
when: When.NotExists,
expiry: _options.IdempotencyWindow)
.ConfigureAwait(false);
if (!stored)
{
// Duplicate enqueue delete the freshly added entry and surface cached ID.
await db.StreamDeleteAsync(
_options.StreamName,
new RedisValue[] { messageId })
.ConfigureAwait(false);
var existing = await db.StringGetAsync(idempotencyKey).ConfigureAwait(false);
var duplicateId = existing.IsNullOrEmpty ? messageId : existing;
_logger.LogDebug(
"Duplicate queue enqueue detected for job {JobId} (token {Token}), returning existing stream id {StreamId}.",
message.JobId,
idempotencyToken,
duplicateId.ToString());
QueueMetrics.RecordDeduplicated(TransportName);
return new QueueEnqueueResult(duplicateId.ToString()!, true);
}
_logger.LogDebug(
"Enqueued job {JobId} into stream {Stream} with id {StreamId}.",
message.JobId,
_options.StreamName,
messageId.ToString());
QueueMetrics.RecordEnqueued(TransportName);
return new QueueEnqueueResult(messageId.ToString()!, false);
}
public async ValueTask<IReadOnlyList<IScanQueueLease>> LeaseAsync(
QueueLeaseRequest request,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
cancellationToken.ThrowIfCancellationRequested();
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
var entries = await db.StreamReadGroupAsync(
_options.StreamName,
_options.ConsumerGroup,
request.Consumer,
position: ">",
count: request.BatchSize,
flags: CommandFlags.None)
.ConfigureAwait(false);
if (entries is null || entries.Length == 0)
{
return Array.Empty<IScanQueueLease>();
}
var now = _timeProvider.GetUtcNow();
var leases = new List<IScanQueueLease>(entries.Length);
foreach (var entry in entries)
{
var lease = TryMapLease(
entry,
request.Consumer,
now,
request.LeaseDuration,
default);
if (lease is null)
{
_logger.LogWarning(
"Stream entry {StreamId} is missing required metadata; acknowledging to avoid poison message.",
entry.Id.ToString());
await db.StreamAcknowledgeAsync(
_options.StreamName,
_options.ConsumerGroup,
new RedisValue[] { entry.Id })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
_options.StreamName,
new RedisValue[] { entry.Id })
.ConfigureAwait(false);
continue;
}
leases.Add(lease);
}
return leases;
}
public async ValueTask<IReadOnlyList<IScanQueueLease>> ClaimExpiredLeasesAsync(
QueueClaimOptions options,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
cancellationToken.ThrowIfCancellationRequested();
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false);
var pending = await db.StreamPendingMessagesAsync(
_options.StreamName,
_options.ConsumerGroup,
options.BatchSize,
RedisValue.Null,
(long)options.MinIdleTime.TotalMilliseconds)
.ConfigureAwait(false);
if (pending is null || pending.Length == 0)
{
return Array.Empty<IScanQueueLease>();
}
var eligible = pending
.Where(p => p.IdleTimeInMilliseconds >= options.MinIdleTime.TotalMilliseconds)
.ToArray();
if (eligible.Length == 0)
{
return Array.Empty<IScanQueueLease>();
}
var messageIds = eligible
.Select(static p => (RedisValue)p.MessageId)
.ToArray();
var entries = await db.StreamClaimAsync(
_options.StreamName,
_options.ConsumerGroup,
options.ClaimantConsumer,
0,
messageIds,
CommandFlags.None)
.ConfigureAwait(false);
if (entries is null || entries.Length == 0)
{
return Array.Empty<IScanQueueLease>();
}
var now = _timeProvider.GetUtcNow();
var pendingById = Enumerable.ToDictionary<StreamPendingMessageInfo, string, StreamPendingMessageInfo>(
eligible,
static p => p.MessageId.IsNullOrEmpty ? string.Empty : p.MessageId.ToString(),
static p => p,
StringComparer.Ordinal);
var leases = new List<IScanQueueLease>(entries.Length);
foreach (var entry in entries)
{
var entryIdValue = entry.Id;
var entryId = entryIdValue.IsNullOrEmpty ? string.Empty : entryIdValue.ToString();
var hasPending = pendingById.TryGetValue(entryId, out var pendingInfo);
var attempt = hasPending
? (int)Math.Max(1, pendingInfo.DeliveryCount)
: 1;
var lease = TryMapLease(
entry,
options.ClaimantConsumer,
now,
_queueOptions.DefaultLeaseDuration,
attempt);
if (lease is null)
{
_logger.LogWarning(
"Unable to map claimed stream entry {StreamId}; acknowledging to unblock queue.",
entry.Id.ToString());
await db.StreamAcknowledgeAsync(
_options.StreamName,
_options.ConsumerGroup,
new RedisValue[] { entry.Id })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
_options.StreamName,
new RedisValue[] { entry.Id })
.ConfigureAwait(false);
continue;
}
leases.Add(lease);
}
return leases;
}
public async ValueTask DisposeAsync()
{
if (_disposed)
{
return;
}
_disposed = true;
if (_connection is not null)
{
await _connection.CloseAsync();
_connection.Dispose();
}
_connectionLock.Dispose();
_groupInitLock.Dispose();
GC.SuppressFinalize(this);
}
internal async Task AcknowledgeAsync(
RedisScanQueueLease lease,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.StreamAcknowledgeAsync(
_options.StreamName,
_options.ConsumerGroup,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
_options.StreamName,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
_logger.LogDebug(
"Acknowledged job {JobId} ({MessageId}) on consumer {Consumer}.",
lease.JobId,
lease.MessageId,
lease.Consumer);
QueueMetrics.RecordAck(TransportName);
}
internal async Task RenewLeaseAsync(
RedisScanQueueLease lease,
TimeSpan leaseDuration,
CancellationToken cancellationToken)
{
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.StreamClaimAsync(
_options.StreamName,
_options.ConsumerGroup,
lease.Consumer,
0,
new RedisValue[] { lease.MessageId },
CommandFlags.None)
.ConfigureAwait(false);
var expires = _timeProvider.GetUtcNow().Add(leaseDuration);
lease.RefreshLease(expires);
_logger.LogDebug(
"Renewed lease for job {JobId} until {LeaseExpiry:u}.",
lease.JobId,
expires);
}
internal async Task ReleaseAsync(
RedisScanQueueLease lease,
QueueReleaseDisposition disposition,
CancellationToken cancellationToken)
{
if (disposition == QueueReleaseDisposition.Retry
&& lease.Attempt >= _queueOptions.MaxDeliveryAttempts)
{
_logger.LogWarning(
"Job {JobId} reached max delivery attempts ({Attempts}); moving to dead-letter.",
lease.JobId,
lease.Attempt);
await DeadLetterAsync(
lease,
$"max-delivery-attempts:{lease.Attempt}",
cancellationToken).ConfigureAwait(false);
return;
}
if (!lease.TryBeginCompletion())
{
return;
}
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.StreamAcknowledgeAsync(
_options.StreamName,
_options.ConsumerGroup,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
_options.StreamName,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
QueueMetrics.RecordAck(TransportName);
if (disposition == QueueReleaseDisposition.Retry)
{
QueueMetrics.RecordRetry(TransportName);
var delay = CalculateBackoff(lease.Attempt);
if (delay > TimeSpan.Zero)
{
_logger.LogDebug(
"Delaying retry for job {JobId} by {Delay} (attempt {Attempt}).",
lease.JobId,
delay,
lease.Attempt);
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
return;
}
}
var requeueMessage = new ScanQueueMessage(lease.JobId, lease.Payload)
{
IdempotencyKey = lease.IdempotencyKey,
Attributes = lease.Attributes,
TraceId = null
};
var now = _timeProvider.GetUtcNow();
var entries = BuildEntries(requeueMessage, now, lease.Attempt + 1);
await AddToStreamAsync(
db,
_options.StreamName,
entries,
_options.ApproximateMaxLength,
_options.ApproximateMaxLength is not null)
.ConfigureAwait(false);
QueueMetrics.RecordEnqueued(TransportName);
_logger.LogWarning(
"Released job {JobId} for retry (attempt {Attempt}).",
lease.JobId,
lease.Attempt + 1);
}
else
{
_logger.LogInformation(
"Abandoned job {JobId} after {Attempt} attempt(s).",
lease.JobId,
lease.Attempt);
}
}
internal async Task DeadLetterAsync(
RedisScanQueueLease lease,
string reason,
CancellationToken cancellationToken)
{
if (!lease.TryBeginCompletion())
{
return;
}
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.StreamAcknowledgeAsync(
_options.StreamName,
_options.ConsumerGroup,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
await db.StreamDeleteAsync(
_options.StreamName,
new RedisValue[] { lease.MessageId })
.ConfigureAwait(false);
var now = _timeProvider.GetUtcNow();
var entries = BuildEntries(
new ScanQueueMessage(lease.JobId, lease.Payload)
{
IdempotencyKey = lease.IdempotencyKey,
Attributes = lease.Attributes,
TraceId = null
},
now,
lease.Attempt);
await AddToStreamAsync(
db,
_queueOptions.DeadLetter.StreamName,
entries,
null,
false)
.ConfigureAwait(false);
_logger.LogError(
"Dead-lettered job {JobId} (attempt {Attempt}): {Reason}",
lease.JobId,
lease.Attempt,
reason);
QueueMetrics.RecordDeadLetter(TransportName);
}
private async ValueTask<IDatabase> GetDatabaseAsync(CancellationToken cancellationToken)
{
if (_connection is not null)
{
return _connection.GetDatabase(_options.Database ?? -1);
}
await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_connection is null)
{
var config = ConfigurationOptions.Parse(_options.ConnectionString!);
config.AbortOnConnectFail = false;
config.ConnectTimeout = (int)_options.InitializationTimeout.TotalMilliseconds;
config.ConnectRetry = 3;
if (_options.Database is not null)
{
config.DefaultDatabase = _options.Database;
}
_connection = await _connectionFactory(config).ConfigureAwait(false);
}
return _connection.GetDatabase(_options.Database ?? -1);
}
finally
{
_connectionLock.Release();
}
}
private async Task EnsureConsumerGroupAsync(
IDatabase database,
CancellationToken cancellationToken)
{
if (_groupInitialized)
{
return;
}
await _groupInitLock.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (_groupInitialized)
{
return;
}
try
{
await database.StreamCreateConsumerGroupAsync(
_options.StreamName,
_options.ConsumerGroup,
StreamPosition.Beginning,
createStream: true)
.ConfigureAwait(false);
}
catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase))
{
// Already exists.
}
_groupInitialized = true;
}
finally
{
_groupInitLock.Release();
}
}
private NameValueEntry[] BuildEntries(
ScanQueueMessage message,
DateTimeOffset enqueuedAt,
int attempt)
{
var attributeCount = message.Attributes?.Count ?? 0;
var entries = ArrayPool<NameValueEntry>.Shared.Rent(6 + attributeCount);
var index = 0;
entries[index++] = new NameValueEntry(QueueEnvelopeFields.JobId, message.JobId);
entries[index++] = new NameValueEntry(QueueEnvelopeFields.Attempt, attempt);
entries[index++] = new NameValueEntry(QueueEnvelopeFields.EnqueuedAt, enqueuedAt.ToUnixTimeMilliseconds());
entries[index++] = new NameValueEntry(
QueueEnvelopeFields.IdempotencyKey,
message.IdempotencyKey ?? message.JobId);
entries[index++] = new NameValueEntry(
QueueEnvelopeFields.Payload,
message.Payload.ToArray());
entries[index++] = new NameValueEntry(
QueueEnvelopeFields.TraceId,
message.TraceId ?? string.Empty);
if (attributeCount > 0)
{
foreach (var kvp in message.Attributes!)
{
entries[index++] = new NameValueEntry(
QueueEnvelopeFields.AttributePrefix + kvp.Key,
kvp.Value);
}
}
var result = entries.AsSpan(0, index).ToArray();
ArrayPool<NameValueEntry>.Shared.Return(entries, clearArray: true);
return result;
}
private RedisScanQueueLease? TryMapLease(
StreamEntry entry,
string consumer,
DateTimeOffset now,
TimeSpan leaseDuration,
int? attemptOverride)
{
if (entry.Values is null || entry.Values.Length == 0)
{
return null;
}
string? jobId = null;
string? idempotency = null;
long? enqueuedAtUnix = null;
byte[]? payload = null;
string? traceId = null;
var attributes = new Dictionary<string, string>(StringComparer.Ordinal);
var attempt = attemptOverride ?? 1;
foreach (var field in entry.Values)
{
var name = field.Name.ToString();
if (name.Equals(QueueEnvelopeFields.JobId, StringComparison.Ordinal))
{
jobId = field.Value.ToString();
}
else if (name.Equals(QueueEnvelopeFields.IdempotencyKey, StringComparison.Ordinal))
{
idempotency = field.Value.ToString();
}
else if (name.Equals(QueueEnvelopeFields.EnqueuedAt, StringComparison.Ordinal))
{
if (long.TryParse(field.Value.ToString(), out var unix))
{
enqueuedAtUnix = unix;
}
}
else if (name.Equals(QueueEnvelopeFields.Payload, StringComparison.Ordinal))
{
payload = (byte[]?)field.Value ?? Array.Empty<byte>();
}
else if (name.Equals(QueueEnvelopeFields.Attempt, StringComparison.Ordinal))
{
if (int.TryParse(field.Value.ToString(), out var parsedAttempt))
{
attempt = Math.Max(parsedAttempt, attempt);
}
}
else if (name.Equals(QueueEnvelopeFields.TraceId, StringComparison.Ordinal))
{
traceId = field.Value.ToString();
}
else if (name.StartsWith(QueueEnvelopeFields.AttributePrefix, StringComparison.Ordinal))
{
attributes[name[QueueEnvelopeFields.AttributePrefix.Length..]] = field.Value.ToString();
}
}
if (jobId is null || payload is null || enqueuedAtUnix is null)
{
return null;
}
var enqueuedAt = DateTimeOffset.FromUnixTimeMilliseconds(enqueuedAtUnix.Value);
var leaseExpires = now.Add(leaseDuration);
var attributeView = attributes.Count == 0
? EmptyReadOnlyDictionary<string, string>.Instance
: new ReadOnlyDictionary<string, string>(attributes);
return new RedisScanQueueLease(
this,
entry.Id.ToString(),
jobId,
payload,
attempt,
enqueuedAt,
leaseExpires,
consumer,
idempotency,
attributeView);
}
private TimeSpan CalculateBackoff(int attempt)
{
var configuredInitial = _options.RetryInitialBackoff > TimeSpan.Zero
? _options.RetryInitialBackoff
: _queueOptions.RetryInitialBackoff;
var initial = configuredInitial > TimeSpan.Zero
? configuredInitial
: TimeSpan.Zero;
if (initial <= TimeSpan.Zero)
{
return TimeSpan.Zero;
}
if (attempt <= 1)
{
return initial;
}
var configuredMax = _queueOptions.RetryMaxBackoff > TimeSpan.Zero
? _queueOptions.RetryMaxBackoff
: initial;
var max = configuredMax <= TimeSpan.Zero
? initial
: configuredMax;
var exponent = attempt - 1;
var scale = Math.Pow(2, exponent - 1);
var scaledTicks = initial.Ticks * scale;
var cappedTicks = Math.Min(max.Ticks, scaledTicks);
var resultTicks = Math.Max(initial.Ticks, (long)cappedTicks);
return TimeSpan.FromTicks(resultTicks);
}
private async Task<RedisValue> AddToStreamAsync(
IDatabase database,
RedisKey stream,
NameValueEntry[] entries,
int? maxLength,
bool useApproximateLength)
{
var capacity = 4 + (entries.Length * 2);
var args = new List<object>(capacity)
{
stream
};
if (maxLength.HasValue)
{
args.Add("MAXLEN");
if (useApproximateLength)
{
args.Add("~");
}
args.Add(maxLength.Value);
}
args.Add("*");
for (var i = 0; i < entries.Length; i++)
{
args.Add(entries[i].Name);
args.Add(entries[i].Value);
}
var result = await database.ExecuteAsync("XADD", args.ToArray()).ConfigureAwait(false);
return (RedisValue)result!;
}
private static class EmptyReadOnlyDictionary<TKey, TValue>
where TKey : notnull
{
public static readonly IReadOnlyDictionary<TKey, TValue> Instance =
new ReadOnlyDictionary<TKey, TValue>(new Dictionary<TKey, TValue>(0, EqualityComparer<TKey>.Default));
}
internal async ValueTask PingAsync(CancellationToken cancellationToken)
{
var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false);
await db.ExecuteAsync("PING").ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,72 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Queue.Redis;
internal sealed class RedisScanQueueLease : IScanQueueLease
{
private readonly RedisScanQueue _queue;
private int _completed;
internal RedisScanQueueLease(
RedisScanQueue queue,
string messageId,
string jobId,
byte[] payload,
int attempt,
DateTimeOffset enqueuedAt,
DateTimeOffset leaseExpiresAt,
string consumer,
string? idempotencyKey,
IReadOnlyDictionary<string, string> attributes)
{
_queue = queue;
MessageId = messageId;
JobId = jobId;
Payload = payload;
Attempt = attempt;
EnqueuedAt = enqueuedAt;
LeaseExpiresAt = leaseExpiresAt;
Consumer = consumer;
IdempotencyKey = idempotencyKey;
Attributes = attributes;
}
public string MessageId { get; }
public string JobId { get; }
public ReadOnlyMemory<byte> Payload { get; }
public int Attempt { get; }
public DateTimeOffset EnqueuedAt { get; }
public DateTimeOffset LeaseExpiresAt { get; private set; }
public string Consumer { get; }
public string? IdempotencyKey { get; }
public IReadOnlyDictionary<string, string> Attributes { get; }
public Task AcknowledgeAsync(CancellationToken cancellationToken = default)
=> _queue.AcknowledgeAsync(this, cancellationToken);
public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default)
=> _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken);
public Task ReleaseAsync(QueueReleaseDisposition disposition, CancellationToken cancellationToken = default)
=> _queue.ReleaseAsync(this, disposition, cancellationToken);
public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default)
=> _queue.DeadLetterAsync(this, reason, cancellationToken);
internal bool TryBeginCompletion()
=> Interlocked.CompareExchange(ref _completed, 1, 0) == 0;
internal void RefreshLease(DateTimeOffset expiresAt)
=> LeaseExpiresAt = expiresAt;
}

View File

@@ -0,0 +1,115 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Scanner.Queue;
public sealed class ScanQueueMessage
{
private readonly byte[] _payload;
public ScanQueueMessage(string jobId, ReadOnlyMemory<byte> payload)
{
if (string.IsNullOrWhiteSpace(jobId))
{
throw new ArgumentException("Job identifier must be provided.", nameof(jobId));
}
JobId = jobId;
_payload = CopyPayload(payload);
}
public string JobId { get; }
public string? IdempotencyKey { get; init; }
public string? TraceId { get; init; }
public IReadOnlyDictionary<string, string>? Attributes { get; init; }
public ReadOnlyMemory<byte> Payload => _payload;
private static byte[] CopyPayload(ReadOnlyMemory<byte> payload)
{
if (payload.Length == 0)
{
return Array.Empty<byte>();
}
var copy = new byte[payload.Length];
payload.Span.CopyTo(copy);
return copy;
}
}
public readonly record struct QueueEnqueueResult(string MessageId, bool Deduplicated);
public sealed class QueueLeaseRequest
{
public QueueLeaseRequest(string consumer, int batchSize, TimeSpan leaseDuration)
{
if (string.IsNullOrWhiteSpace(consumer))
{
throw new ArgumentException("Consumer name must be provided.", nameof(consumer));
}
if (batchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive.");
}
if (leaseDuration <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(leaseDuration), leaseDuration, "Lease duration must be positive.");
}
Consumer = consumer;
BatchSize = batchSize;
LeaseDuration = leaseDuration;
}
public string Consumer { get; }
public int BatchSize { get; }
public TimeSpan LeaseDuration { get; }
}
public sealed class QueueClaimOptions
{
public QueueClaimOptions(
string claimantConsumer,
int batchSize,
TimeSpan minIdleTime)
{
if (string.IsNullOrWhiteSpace(claimantConsumer))
{
throw new ArgumentException("Consumer must be provided.", nameof(claimantConsumer));
}
if (batchSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive.");
}
if (minIdleTime < TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(nameof(minIdleTime), minIdleTime, "Idle time cannot be negative.");
}
ClaimantConsumer = claimantConsumer;
BatchSize = batchSize;
MinIdleTime = minIdleTime;
}
public string ClaimantConsumer { get; }
public int BatchSize { get; }
public TimeSpan MinIdleTime { get; }
}
public enum QueueReleaseDisposition
{
Retry,
Abandon
}

View File

@@ -0,0 +1,55 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Queue.Nats;
using StellaOps.Scanner.Queue.Redis;
namespace StellaOps.Scanner.Queue;
public sealed class ScannerQueueHealthCheck : IHealthCheck
{
private readonly IScanQueue _queue;
private readonly ILogger<ScannerQueueHealthCheck> _logger;
public ScannerQueueHealthCheck(
IScanQueue queue,
ILogger<ScannerQueueHealthCheck> logger)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
switch (_queue)
{
case RedisScanQueue redisQueue:
await redisQueue.PingAsync(cancellationToken).ConfigureAwait(false);
return HealthCheckResult.Healthy("Redis queue reachable.");
case NatsScanQueue natsQueue:
await natsQueue.PingAsync(cancellationToken).ConfigureAwait(false);
return HealthCheckResult.Healthy("NATS queue reachable.");
default:
return HealthCheckResult.Healthy("Queue transport without dedicated ping returned healthy.");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Scanner queue health check failed.");
return new HealthCheckResult(
context.Registration.FailureStatus,
"Queue transport unreachable.",
ex);
}
}
}

View File

@@ -0,0 +1,92 @@
using System;
namespace StellaOps.Scanner.Queue;
public sealed class ScannerQueueOptions
{
public QueueTransportKind Kind { get; set; } = QueueTransportKind.Redis;
public RedisQueueOptions Redis { get; set; } = new();
public NatsQueueOptions Nats { get; set; } = new();
/// <summary>
/// Default lease duration applied when callers do not override the visibility timeout.
/// </summary>
public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Maximum number of times a message may be delivered before it is shunted to the dead-letter queue.
/// </summary>
public int MaxDeliveryAttempts { get; set; } = 5;
/// <summary>
/// Options controlling retry/backoff/dead-letter handling.
/// </summary>
public DeadLetterQueueOptions DeadLetter { get; set; } = new();
/// <summary>
/// Initial backoff applied when a job is retried after failure.
/// </summary>
public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Maximum backoff window applied for exponential retry.
/// </summary>
public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromMinutes(2);
}
public sealed class RedisQueueOptions
{
public string? ConnectionString { get; set; }
public int? Database { get; set; }
public string StreamName { get; set; } = "scanner:jobs";
public string ConsumerGroup { get; set; } = "scanner-workers";
public string IdempotencyKeyPrefix { get; set; } = "scanner:jobs:idemp:";
public TimeSpan IdempotencyWindow { get; set; } = TimeSpan.FromHours(12);
public int? ApproximateMaxLength { get; set; }
public TimeSpan InitializationTimeout { get; set; } = TimeSpan.FromSeconds(30);
public TimeSpan ClaimIdleThreshold { get; set; } = TimeSpan.FromMinutes(10);
public TimeSpan PendingScanWindow { get; set; } = TimeSpan.FromMinutes(30);
public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5);
}
public sealed class NatsQueueOptions
{
public string? Url { get; set; }
public string Stream { get; set; } = "SCANNER_JOBS";
public string Subject { get; set; } = "scanner.jobs";
public string DurableConsumer { get; set; } = "scanner-workers";
public int MaxInFlight { get; set; } = 64;
public TimeSpan AckWait { get; set; } = TimeSpan.FromMinutes(5);
public string DeadLetterStream { get; set; } = "SCANNER_JOBS_DEAD";
public string DeadLetterSubject { get; set; } = "scanner.jobs.dead";
public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(10);
public TimeSpan IdleHeartbeat { get; set; } = TimeSpan.FromSeconds(30);
}
public sealed class DeadLetterQueueOptions
{
public string StreamName { get; set; } = "scanner:jobs:dead";
public TimeSpan Retention { get; set; } = TimeSpan.FromDays(7);
}

View File

@@ -0,0 +1,67 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Queue.Nats;
using StellaOps.Scanner.Queue.Redis;
namespace StellaOps.Scanner.Queue;
public static class ScannerQueueServiceCollectionExtensions
{
public static IServiceCollection AddScannerQueue(
this IServiceCollection services,
IConfiguration configuration,
string sectionName = "scanner:queue")
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
var options = new ScannerQueueOptions();
configuration.GetSection(sectionName).Bind(options);
services.TryAddSingleton(TimeProvider.System);
services.AddSingleton(options);
services.AddSingleton<IScanQueue>(sp =>
{
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
var timeProvider = sp.GetService<TimeProvider>() ?? TimeProvider.System;
return options.Kind switch
{
QueueTransportKind.Redis => new RedisScanQueue(
options,
options.Redis,
loggerFactory.CreateLogger<RedisScanQueue>(),
timeProvider),
QueueTransportKind.Nats => new NatsScanQueue(
options,
options.Nats,
loggerFactory.CreateLogger<NatsScanQueue>(),
timeProvider),
_ => throw new InvalidOperationException($"Unsupported queue transport kind '{options.Kind}'.")
};
});
services.AddSingleton<ScannerQueueHealthCheck>();
return services;
}
public static IHealthChecksBuilder AddScannerQueueHealthCheck(
this IHealthChecksBuilder builder)
{
ArgumentNullException.ThrowIfNull(builder);
builder.Services.TryAddSingleton<ScannerQueueHealthCheck>();
builder.AddCheck<ScannerQueueHealthCheck>(
name: "scanner-queue",
failureStatus: HealthStatus.Unhealthy,
tags: new[] { "scanner", "queue" });
return builder;
}
}

View File

@@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.1" />
<PackageReference Include="StackExchange.Redis" Version="2.7.33" />
<PackageReference Include="NATS.Client.Core" Version="2.0.0" />
<PackageReference Include="NATS.Client.JetStream" Version="2.0.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
# Scanner Queue Task Board (Sprint 9)
| ID | Status | Owner(s) | Depends on | Description | Exit Criteria |
|----|--------|----------|------------|-------------|---------------|
| SCANNER-QUEUE-09-401 | DONE (2025-10-19) | Scanner Queue Guild | — | Implement queue abstraction + Redis Streams adapter with ack/lease semantics, idempotency tokens, and deterministic job IDs. | Interfaces finalized; Redis adapter passes enqueue/dequeue/ack/claim lease tests; structured logs exercised. |
| SCANNER-QUEUE-09-402 | DONE (2025-10-19) | Scanner Queue Guild | SCANNER-QUEUE-09-401 | Add pluggable backend support (Redis, NATS) with configuration binding, health probes, failover documentation. | NATS adapter + DI bindings delivered; health checks documented; configuration tests green. |
| SCANNER-QUEUE-09-403 | DONE (2025-10-19) | Scanner Queue Guild | SCANNER-QUEUE-09-401 | Implement retry and dead-letter flow with structured metrics/logs for offline deployments. | Retry policy configurable; dead-letter queue persisted; metrics counters validated in integration tests. |

View File

@@ -0,0 +1,82 @@
using System;
using System.Net;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scanner.Sbomer.BuildXPlugin;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
using Xunit;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Attestation;
public sealed class AttestorClientTests
{
[Fact]
public async Task SendPlaceholderAsync_PostsJsonPayload()
{
var handler = new RecordingHandler(new HttpResponseMessage(HttpStatusCode.Accepted));
using var httpClient = new HttpClient(handler);
var client = new AttestorClient(httpClient);
var document = BuildDescriptorDocument();
var attestorUri = new Uri("https://attestor.example.com/api/v1/provenance");
await client.SendPlaceholderAsync(attestorUri, document, CancellationToken.None);
Assert.NotNull(handler.CapturedRequest);
Assert.Equal(HttpMethod.Post, handler.CapturedRequest!.Method);
Assert.Equal(attestorUri, handler.CapturedRequest.RequestUri);
var content = await handler.CapturedRequest.Content!.ReadAsStringAsync();
var json = JsonDocument.Parse(content);
Assert.Equal(document.Subject.Digest, json.RootElement.GetProperty("imageDigest").GetString());
Assert.Equal(document.Artifact.Digest, json.RootElement.GetProperty("sbomDigest").GetString());
Assert.Equal(document.Provenance.ExpectedDsseSha256, json.RootElement.GetProperty("expectedDsseSha256").GetString());
}
[Fact]
public async Task SendPlaceholderAsync_ThrowsOnFailure()
{
var handler = new RecordingHandler(new HttpResponseMessage(HttpStatusCode.BadRequest)
{
Content = new StringContent("invalid")
});
using var httpClient = new HttpClient(handler);
var client = new AttestorClient(httpClient);
var document = BuildDescriptorDocument();
var attestorUri = new Uri("https://attestor.example.com/api/v1/provenance");
await Assert.ThrowsAsync<BuildxPluginException>(() => client.SendPlaceholderAsync(attestorUri, document, CancellationToken.None));
}
private static DescriptorDocument BuildDescriptorDocument()
{
var subject = new DescriptorSubject("application/vnd.oci.image.manifest.v1+json", "sha256:img");
var artifact = new DescriptorArtifact("application/vnd.cyclonedx+json", "sha256:sbom", 42, new System.Collections.Generic.Dictionary<string, string>());
var provenance = new DescriptorProvenance("pending", "sha256:dsse", "nonce", "https://attestor.example.com/api/v1/provenance", "https://slsa.dev/provenance/v1");
var generatorMetadata = new DescriptorGeneratorMetadata("generator", "1.0.0");
var metadata = new System.Collections.Generic.Dictionary<string, string>();
return new DescriptorDocument("schema", DateTimeOffset.UtcNow, generatorMetadata, subject, artifact, provenance, metadata);
}
private sealed class RecordingHandler : HttpMessageHandler
{
private readonly HttpResponseMessage response;
public RecordingHandler(HttpResponseMessage response)
{
this.response = response;
}
public HttpRequestMessage? CapturedRequest { get; private set; }
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
CapturedRequest = request;
return Task.FromResult(response);
}
}
}

View File

@@ -0,0 +1,34 @@
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Cas;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities;
using Xunit;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Cas;
public sealed class LocalCasClientTests
{
[Fact]
public async Task VerifyWriteAsync_WritesProbeObject()
{
await using var temp = new TempDirectory();
var client = new LocalCasClient(new LocalCasOptions
{
RootDirectory = temp.Path,
Algorithm = "sha256"
});
var result = await client.VerifyWriteAsync(CancellationToken.None);
Assert.Equal("sha256", result.Algorithm);
Assert.True(File.Exists(result.Path));
var bytes = await File.ReadAllBytesAsync(result.Path);
Assert.Equal("stellaops-buildx-probe"u8.ToArray(), bytes);
var expectedDigest = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
Assert.Equal(expectedDigest, result.Digest);
}
}

View File

@@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities;
using Xunit;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Descriptor;
public sealed class DescriptorGeneratorTests
{
[Fact]
public async Task CreateAsync_BuildsDeterministicDescriptor()
{
await using var temp = new TempDirectory();
var sbomPath = Path.Combine(temp.Path, "sample.cdx.json");
await File.WriteAllTextAsync(sbomPath, "{\"bomFormat\":\"CycloneDX\",\"specVersion\":\"1.5\"}");
var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero));
var generator = new DescriptorGenerator(fakeTime);
var request = new DescriptorRequest
{
ImageDigest = "sha256:0123456789abcdef",
SbomPath = sbomPath,
SbomMediaType = "application/vnd.cyclonedx+json",
SbomFormat = "cyclonedx-json",
SbomKind = "inventory",
SbomArtifactType = "application/vnd.stellaops.sbom.layer+json",
SubjectMediaType = "application/vnd.oci.image.manifest.v1+json",
GeneratorVersion = "1.2.3",
GeneratorName = "StellaOps.Scanner.Sbomer.BuildXPlugin",
LicenseId = "lic-123",
SbomName = "sample.cdx.json",
Repository = "git.stella-ops.org/stellaops",
BuildRef = "refs/heads/main",
AttestorUri = "https://attestor.local/api/v1/provenance"
}.Validate();
var document = await generator.CreateAsync(request, CancellationToken.None);
Assert.Equal(DescriptorGenerator.Schema, document.Schema);
Assert.Equal(fakeTime.GetUtcNow(), document.GeneratedAt);
Assert.Equal(request.ImageDigest, document.Subject.Digest);
Assert.Equal(request.SbomMediaType, document.Artifact.MediaType);
Assert.Equal(request.SbomName, document.Artifact.Annotations["org.opencontainers.image.title"]);
Assert.Equal("pending", document.Provenance.Status);
Assert.Equal(request.AttestorUri, document.Provenance.AttestorUri);
Assert.Equal(request.PredicateType, document.Provenance.PredicateType);
var expectedSbomDigest = ComputeSha256File(sbomPath);
Assert.Equal(expectedSbomDigest, document.Artifact.Digest);
Assert.Equal(expectedSbomDigest, document.Metadata["sbomDigest"]);
var expectedDsse = ComputeExpectedDsse(request.ImageDigest, expectedSbomDigest, document.Provenance.Nonce);
Assert.Equal(expectedDsse, document.Provenance.ExpectedDsseSha256);
Assert.Equal(expectedDsse, document.Artifact.Annotations["org.stellaops.provenance.dsse.sha256"]);
}
private static string ComputeSha256File(string path)
{
using var stream = File.OpenRead(path);
var hash = SHA256.HashData(stream);
return $"sha256:{Convert.ToHexString(hash).ToLower(CultureInfo.InvariantCulture)}";
}
private static string ComputeExpectedDsse(string imageDigest, string sbomDigest, string nonce)
{
var payload = $"{imageDigest}\n{sbomDigest}\n{nonce}";
Span<byte> hash = stackalloc byte[32];
SHA256.HashData(Encoding.UTF8.GetBytes(payload), hash);
return $"sha256:{Convert.ToHexString(hash).ToLower(CultureInfo.InvariantCulture)}";
}
}

View File

@@ -0,0 +1,80 @@
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scanner.Sbomer.BuildXPlugin;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities;
using Xunit;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Manifest;
public sealed class BuildxPluginManifestLoaderTests
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = true
};
[Fact]
public async Task LoadAsync_ReturnsManifestWithSourceInformation()
{
await using var temp = new TempDirectory();
var manifestPath = System.IO.Path.Combine(temp.Path, "stellaops.manifest.json");
await File.WriteAllTextAsync(manifestPath, BuildSampleManifestJson("stellaops.sbom-indexer"));
var loader = new BuildxPluginManifestLoader(temp.Path);
var manifests = await loader.LoadAsync(CancellationToken.None);
var manifest = Assert.Single(manifests);
Assert.Equal("stellaops.sbom-indexer", manifest.Id);
Assert.Equal("0.1.0", manifest.Version);
Assert.Equal(manifestPath, manifest.SourcePath);
Assert.Equal(Path.GetDirectoryName(manifestPath), manifest.SourceDirectory);
}
[Fact]
public async Task LoadDefaultAsync_ThrowsWhenNoManifests()
{
await using var temp = new TempDirectory();
var loader = new BuildxPluginManifestLoader(temp.Path);
await Assert.ThrowsAsync<BuildxPluginException>(() => loader.LoadDefaultAsync(CancellationToken.None));
}
[Fact]
public async Task LoadAsync_ThrowsWhenRestartRequiredMissing()
{
await using var temp = new TempDirectory();
var manifestPath = Path.Combine(temp.Path, "failure.manifest.json");
await File.WriteAllTextAsync(manifestPath, BuildSampleManifestJson("stellaops.failure", requiresRestart: false));
var loader = new BuildxPluginManifestLoader(temp.Path);
await Assert.ThrowsAsync<BuildxPluginException>(() => loader.LoadAsync(CancellationToken.None));
}
private static string BuildSampleManifestJson(string id, bool requiresRestart = true)
{
var manifest = new BuildxPluginManifest
{
SchemaVersion = BuildxPluginManifest.CurrentSchemaVersion,
Id = id,
DisplayName = "Sample",
Version = "0.1.0",
RequiresRestart = requiresRestart,
EntryPoint = new BuildxPluginEntryPoint
{
Type = "dotnet",
Executable = "StellaOps.Scanner.Sbomer.BuildXPlugin.dll"
},
Cas = new BuildxPluginCas
{
Protocol = "filesystem",
DefaultRoot = "cas"
}
};
return JsonSerializer.Serialize(manifest, SerializerOptions);
}
}

View File

@@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Scanner.Sbomer.BuildXPlugin\StellaOps.Scanner.Sbomer.BuildXPlugin.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,44 @@
using System;
using System.IO;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities;
internal sealed class TempDirectory : IDisposable, IAsyncDisposable
{
public string Path { get; }
public TempDirectory()
{
Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-buildx-{Guid.NewGuid():N}");
Directory.CreateDirectory(Path);
}
public void Dispose()
{
Cleanup();
GC.SuppressFinalize(this);
}
public ValueTask DisposeAsync()
{
Cleanup();
GC.SuppressFinalize(this);
return ValueTask.CompletedTask;
}
private void Cleanup()
{
try
{
if (Directory.Exists(Path))
{
Directory.Delete(Path, recursive: true);
}
}
catch
{
// Best effort cleanup only.
}
}
}

View File

@@ -0,0 +1,49 @@
using System;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation;
/// <summary>
/// Sends provenance placeholders to the Attestor service for asynchronous DSSE signing.
/// </summary>
public sealed class AttestorClient
{
private readonly HttpClient httpClient;
public AttestorClient(HttpClient httpClient)
{
this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
}
public async Task SendPlaceholderAsync(Uri attestorUri, DescriptorDocument document, CancellationToken cancellationToken)
{
if (attestorUri is null)
{
throw new ArgumentNullException(nameof(attestorUri));
}
if (document is null)
{
throw new ArgumentNullException(nameof(document));
}
var payload = new AttestorProvenanceRequest(
ImageDigest: document.Subject.Digest,
SbomDigest: document.Artifact.Digest,
ExpectedDsseSha256: document.Provenance.ExpectedDsseSha256,
Nonce: document.Provenance.Nonce,
PredicateType: document.Provenance.PredicateType,
Schema: document.Schema);
using var response = await httpClient.PostAsJsonAsync(attestorUri, payload, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
throw new BuildxPluginException($"Attestor rejected provenance placeholder ({(int)response.StatusCode}): {body}");
}
}
}

View File

@@ -0,0 +1,11 @@
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation;
public sealed record AttestorProvenanceRequest(
[property: JsonPropertyName("imageDigest")] string ImageDigest,
[property: JsonPropertyName("sbomDigest")] string SbomDigest,
[property: JsonPropertyName("expectedDsseSha256")] string ExpectedDsseSha256,
[property: JsonPropertyName("nonce")] string Nonce,
[property: JsonPropertyName("predicateType")] string PredicateType,
[property: JsonPropertyName("schema")] string Schema);

View File

@@ -0,0 +1,19 @@
using System;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin;
/// <summary>
/// Represents user-facing errors raised by the BuildX plug-in.
/// </summary>
public sealed class BuildxPluginException : Exception
{
public BuildxPluginException(string message)
: base(message)
{
}
public BuildxPluginException(string message, Exception innerException)
: base(message, innerException)
{
}
}

View File

@@ -0,0 +1,6 @@
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Cas;
/// <summary>
/// Result of persisting bytes into the local CAS.
/// </summary>
public sealed record CasWriteResult(string Algorithm, string Digest, string Path);

View File

@@ -0,0 +1,74 @@
using System;
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Cas;
/// <summary>
/// Minimal filesystem-backed CAS used when the BuildX generator runs inside CI.
/// </summary>
public sealed class LocalCasClient
{
private readonly string rootDirectory;
private readonly string algorithm;
public LocalCasClient(LocalCasOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
algorithm = options.Algorithm.ToLowerInvariant();
if (!string.Equals(algorithm, "sha256", StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("Only the sha256 algorithm is supported.", nameof(options));
}
rootDirectory = Path.GetFullPath(options.RootDirectory);
}
public Task<CasWriteResult> VerifyWriteAsync(CancellationToken cancellationToken)
{
ReadOnlyMemory<byte> probe = "stellaops-buildx-probe"u8.ToArray();
return WriteAsync(probe, cancellationToken);
}
public async Task<CasWriteResult> WriteAsync(ReadOnlyMemory<byte> content, CancellationToken cancellationToken)
{
var digest = ComputeDigest(content.Span);
var path = BuildObjectPath(digest);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
await using var stream = new FileStream(
path,
FileMode.Create,
FileAccess.Write,
FileShare.Read,
bufferSize: 16 * 1024,
FileOptions.Asynchronous | FileOptions.SequentialScan);
await stream.WriteAsync(content, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
return new CasWriteResult(algorithm, digest, path);
}
private string BuildObjectPath(string digest)
{
// Layout: <root>/<algorithm>/<first two>/<rest>.bin
var prefix = digest.Substring(0, 2);
var suffix = digest[2..];
return Path.Combine(rootDirectory, algorithm, prefix, $"{suffix}.bin");
}
private static string ComputeDigest(ReadOnlySpan<byte> content)
{
Span<byte> buffer = stackalloc byte[32];
SHA256.HashData(content, buffer);
return Convert.ToHexString(buffer).ToLowerInvariant();
}
}

View File

@@ -0,0 +1,40 @@
using System;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Cas;
/// <summary>
/// Configuration for the on-disk content-addressable store used during CI.
/// </summary>
public sealed record LocalCasOptions
{
private string rootDirectory = string.Empty;
private string algorithm = "sha256";
public string RootDirectory
{
get => rootDirectory;
init
{
if (string.IsNullOrWhiteSpace(value))
{
throw new ArgumentException("Root directory must be provided.", nameof(value));
}
rootDirectory = value;
}
}
public string Algorithm
{
get => algorithm;
init
{
if (string.IsNullOrWhiteSpace(value))
{
throw new ArgumentException("Algorithm must be provided.", nameof(value));
}
algorithm = value;
}
}
}

View File

@@ -0,0 +1,13 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
/// <summary>
/// Represents an OCI artifact descriptor emitted by the BuildX generator.
/// </summary>
public sealed record DescriptorArtifact(
[property: JsonPropertyName("mediaType")] string MediaType,
[property: JsonPropertyName("digest")] string Digest,
[property: JsonPropertyName("size")] long Size,
[property: JsonPropertyName("annotations")] IReadOnlyDictionary<string, string> Annotations);

View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
/// <summary>
/// Root payload describing BuildX generator output with provenance placeholders.
/// </summary>
public sealed record DescriptorDocument(
[property: JsonPropertyName("schema")] string Schema,
[property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt,
[property: JsonPropertyName("generator")] DescriptorGeneratorMetadata Generator,
[property: JsonPropertyName("subject")] DescriptorSubject Subject,
[property: JsonPropertyName("artifact")] DescriptorArtifact Artifact,
[property: JsonPropertyName("provenance")] DescriptorProvenance Provenance,
[property: JsonPropertyName("metadata")] IReadOnlyDictionary<string, string> Metadata);

View File

@@ -0,0 +1,180 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
/// <summary>
/// Builds immutable OCI descriptors enriched with provenance placeholders.
/// </summary>
public sealed class DescriptorGenerator
{
public const string Schema = "stellaops.buildx.descriptor.v1";
private readonly TimeProvider timeProvider;
public DescriptorGenerator(TimeProvider timeProvider)
{
timeProvider ??= TimeProvider.System;
this.timeProvider = timeProvider;
}
public async Task<DescriptorDocument> CreateAsync(DescriptorRequest request, CancellationToken cancellationToken)
{
if (request is null)
{
throw new ArgumentNullException(nameof(request));
}
if (string.IsNullOrWhiteSpace(request.ImageDigest))
{
throw new BuildxPluginException("Image digest must be provided.");
}
if (string.IsNullOrWhiteSpace(request.SbomPath))
{
throw new BuildxPluginException("SBOM path must be provided.");
}
var sbomFile = new FileInfo(request.SbomPath);
if (!sbomFile.Exists)
{
throw new BuildxPluginException($"SBOM file '{request.SbomPath}' was not found.");
}
var sbomDigest = await ComputeFileDigestAsync(sbomFile, cancellationToken).ConfigureAwait(false);
var nonce = Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture);
var expectedDsseSha = ComputeExpectedDsseDigest(request.ImageDigest, sbomDigest, nonce);
var artifactAnnotations = BuildArtifactAnnotations(request, nonce, expectedDsseSha);
var subject = new DescriptorSubject(
MediaType: request.SubjectMediaType,
Digest: request.ImageDigest);
var artifact = new DescriptorArtifact(
MediaType: request.SbomMediaType,
Digest: sbomDigest,
Size: sbomFile.Length,
Annotations: artifactAnnotations);
var provenance = new DescriptorProvenance(
Status: "pending",
ExpectedDsseSha256: expectedDsseSha,
Nonce: nonce,
AttestorUri: request.AttestorUri,
PredicateType: request.PredicateType);
var generatorMetadata = new DescriptorGeneratorMetadata(
Name: request.GeneratorName ?? "StellaOps.Scanner.Sbomer.BuildXPlugin",
Version: request.GeneratorVersion);
var metadata = BuildDocumentMetadata(request, sbomFile, sbomDigest);
return new DescriptorDocument(
Schema: Schema,
GeneratedAt: timeProvider.GetUtcNow(),
Generator: generatorMetadata,
Subject: subject,
Artifact: artifact,
Provenance: provenance,
Metadata: metadata);
}
private static async Task<string> ComputeFileDigestAsync(FileInfo file, CancellationToken cancellationToken)
{
await using var stream = new FileStream(
file.FullName,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
bufferSize: 128 * 1024,
FileOptions.Asynchronous | FileOptions.SequentialScan);
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var buffer = new byte[128 * 1024];
int bytesRead;
while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken).ConfigureAwait(false)) > 0)
{
hash.AppendData(buffer, 0, bytesRead);
}
var digest = hash.GetHashAndReset();
return $"sha256:{Convert.ToHexString(digest).ToLowerInvariant()}";
}
private static string ComputeExpectedDsseDigest(string imageDigest, string sbomDigest, string nonce)
{
var payload = $"{imageDigest}\n{sbomDigest}\n{nonce}";
var bytes = System.Text.Encoding.UTF8.GetBytes(payload);
Span<byte> hash = stackalloc byte[32];
SHA256.HashData(bytes, hash);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private static IReadOnlyDictionary<string, string> BuildArtifactAnnotations(DescriptorRequest request, string nonce, string expectedDsse)
{
var annotations = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["org.opencontainers.artifact.type"] = request.SbomArtifactType,
["org.stellaops.scanner.version"] = request.GeneratorVersion,
["org.stellaops.sbom.kind"] = request.SbomKind,
["org.stellaops.sbom.format"] = request.SbomFormat,
["org.stellaops.provenance.status"] = "pending",
["org.stellaops.provenance.dsse.sha256"] = expectedDsse,
["org.stellaops.provenance.nonce"] = nonce
};
if (!string.IsNullOrWhiteSpace(request.LicenseId))
{
annotations["org.stellaops.license.id"] = request.LicenseId!;
}
if (!string.IsNullOrWhiteSpace(request.SbomName))
{
annotations["org.opencontainers.image.title"] = request.SbomName!;
}
if (!string.IsNullOrWhiteSpace(request.Repository))
{
annotations["org.stellaops.repository"] = request.Repository!;
}
return annotations;
}
private static IReadOnlyDictionary<string, string> BuildDocumentMetadata(DescriptorRequest request, FileInfo fileInfo, string sbomDigest)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["sbomDigest"] = sbomDigest,
["sbomPath"] = fileInfo.FullName,
["sbomMediaType"] = request.SbomMediaType,
["subjectMediaType"] = request.SubjectMediaType
};
if (!string.IsNullOrWhiteSpace(request.Repository))
{
metadata["repository"] = request.Repository!;
}
if (!string.IsNullOrWhiteSpace(request.BuildRef))
{
metadata["buildRef"] = request.BuildRef!;
}
if (!string.IsNullOrWhiteSpace(request.AttestorUri))
{
metadata["attestorUri"] = request.AttestorUri!;
}
return metadata;
}
}

View File

@@ -0,0 +1,7 @@
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor;
public sealed record DescriptorGeneratorMetadata(
[property: JsonPropertyName("name")] string Name,
[property: JsonPropertyName("version")] string Version);

Some files were not shown because too many files have changed in this diff Show More