up
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-01 21:16:22 +02:00
parent c11d87d252
commit 909d9b6220
208 changed files with 860954 additions and 832 deletions

View File

@@ -0,0 +1,30 @@
# Graph Bench Harness (BENCH-GRAPH-21-001)
Purpose: measure basic graph load/adjacency build and shallow path exploration over deterministic fixtures.
## Fixtures
- Use interim synthetic fixtures under `samples/graph/interim/graph-50k` or `graph-100k`.
- Each fixture includes `nodes.ndjson`, `edges.ndjson`, and `manifest.json` with hashes/counts.
## Usage
```bash
python graph_bench.py \
--fixture ../../../samples/graph/interim/graph-50k \
--output results/graph-50k.json \
--samples 100
```
Outputs a JSON summary with:
- `nodes`, `edges`
- `build_ms` — time to build adjacency (ms)
- `bfs_ms` — total time for 3-depth BFS over sampled nodes
- `avg_reach_3`, `max_reach_3` — nodes reached within depth 3
- `manifest` — copied from fixture for traceability
Determinism:
- Sorted node ids, fixed sample size, stable ordering, no randomness beyond fixture content.
- No network access; pure local file reads.
Next steps (after overlay schema lands):
- Extend to load overlay snapshots and measure overlay-join overhead.
- Add p95/median latency over multiple runs and optional concurrency knobs.

View File

@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""
Graph benchmark harness (BENCH-GRAPH-21-001)
Reads deterministic NDJSON fixtures (nodes/edges) and computes basic metrics plus
lightweight path queries to exercise adjacency building. Uses only local files,
no network, and fixed seeds for reproducibility.
"""
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
from typing import Dict, List, Tuple
def load_ndjson(path: Path):
with path.open("r", encoding="utf-8") as f:
for line in f:
if line.strip():
yield json.loads(line)
def build_graph(nodes_path: Path, edges_path: Path) -> Tuple[Dict[str, List[str]], int]:
adjacency: Dict[str, List[str]] = {}
node_set = set()
for n in load_ndjson(nodes_path):
node_set.add(n["id"])
adjacency.setdefault(n["id"], [])
edge_count = 0
for e in load_ndjson(edges_path):
source = e["source"]
target = e["target"]
# Only keep edges where nodes exist
if source in adjacency and target in adjacency:
adjacency[source].append(target)
edge_count += 1
# sort neighbors for determinism
for v in adjacency.values():
v.sort()
return adjacency, edge_count
def bfs_limited(adjacency: Dict[str, List[str]], start: str, max_depth: int = 3) -> int:
visited = {start}
frontier = [start]
for _ in range(max_depth):
next_frontier = []
for node in frontier:
for nbr in adjacency.get(node, []):
if nbr not in visited:
visited.add(nbr)
next_frontier.append(nbr)
if not next_frontier:
break
frontier = next_frontier
return len(visited)
def run_bench(fixture_dir: Path, sample_size: int = 100) -> dict:
nodes_path = fixture_dir / "nodes.ndjson"
edges_path = fixture_dir / "edges.ndjson"
manifest_path = fixture_dir / "manifest.json"
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
t0 = time.perf_counter()
adjacency, edge_count = build_graph(nodes_path, edges_path)
build_ms = (time.perf_counter() - t0) * 1000
# deterministic sample: first N node ids sorted
node_ids = sorted(adjacency.keys())[:sample_size]
reach_counts = []
t1 = time.perf_counter()
for node_id in node_ids:
reach_counts.append(bfs_limited(adjacency, node_id, max_depth=3))
bfs_ms = (time.perf_counter() - t1) * 1000
avg_reach = sum(reach_counts) / len(reach_counts) if reach_counts else 0
max_reach = max(reach_counts) if reach_counts else 0
return {
"fixture": fixture_dir.name,
"nodes": len(adjacency),
"edges": edge_count,
"build_ms": round(build_ms, 2),
"bfs_ms": round(bfs_ms, 2),
"bfs_samples": len(node_ids),
"avg_reach_3": round(avg_reach, 2),
"max_reach_3": max_reach,
"manifest": manifest,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--fixture", required=True, help="Path to fixture directory (nodes.ndjson, edges.ndjson)")
parser.add_argument("--output", required=True, help="Path to write results JSON")
parser.add_argument("--samples", type=int, default=100, help="Number of starting nodes to sample deterministically")
args = parser.parse_args()
fixture_dir = Path(args.fixture).resolve()
out_path = Path(args.output).resolve()
out_path.parent.mkdir(parents=True, exist_ok=True)
result = run_bench(fixture_dir, sample_size=args.samples)
out_path.write_text(json.dumps(result, indent=2, sort_keys=True))
print(f"Wrote results to {out_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -3,3 +3,4 @@
| ID | Status | Sprint | Notes | Evidence |
| --- | --- | --- | --- | --- |
| BENCH-DETERMINISM-401-057 | DONE (2025-11-26) | SPRINT_0512_0001_0001_bench | Determinism harness and mock scanner added under `src/Bench/StellaOps.Bench/Determinism`; manifests + sample inputs included. | `src/Bench/StellaOps.Bench/Determinism/results` (generated) |
| BENCH-GRAPH-21-001 | DOING (2025-12-01) | SPRINT_0512_0001_0001_bench | Added interim graph bench harness (`Graph/graph_bench.py`) using synthetic 50k/100k fixtures; measures adjacency build + depth-3 reach; pending overlay schema for final fixture integration. | `src/Bench/StellaOps.Bench/Graph` |

View File

@@ -233,10 +233,15 @@ internal static class CommandHandlers
{
var console = AnsiConsole.Console;
console.MarkupLine($"[bold]Scan[/]: {result.ScanId}");
console.MarkupLine($"Image: {result.ImageDigest}");
console.MarkupLine($"Generated: {result.GeneratedAt:O}");
console.MarkupLine($"Outcome: {result.Graph.Outcome}");
console.MarkupLine($"[bold]Scan[/]: {result.ScanId}");
console.MarkupLine($"Image: {result.ImageDigest}");
console.MarkupLine($"Generated: {result.GeneratedAt:O}");
console.MarkupLine($"Outcome: {result.Graph.Outcome}");
if (result.BestPlan is not null)
{
console.MarkupLine($"Best Terminal: {result.BestPlan.TerminalPath} (conf {result.BestPlan.Confidence:F1}, user {result.BestPlan.User}, cwd {result.BestPlan.WorkingDirectory})");
}
var planTable = new Table()
.AddColumn("Terminal")
@@ -248,14 +253,15 @@ internal static class CommandHandlers
foreach (var plan in result.Graph.Plans.OrderByDescending(p => p.Confidence))
{
planTable.AddRow(
plan.TerminalPath,
plan.Runtime ?? "-",
plan.Type.ToString(),
plan.Confidence.ToString("F1", CultureInfo.InvariantCulture),
plan.User,
plan.WorkingDirectory);
}
var confidence = plan.Confidence.ToString("F1", CultureInfo.InvariantCulture);
planTable.AddRow(
plan.TerminalPath,
plan.Runtime ?? "-",
plan.Type.ToString(),
confidence,
plan.User,
plan.WorkingDirectory);
}
if (planTable.Rows.Count > 0)
{

View File

@@ -909,11 +909,11 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient
throw new InvalidOperationException(failure);
}
var result = await response.Content.ReadFromJsonAsync<EntryTraceResponseModel>(SerializerOptions, cancellationToken).ConfigureAwait(false);
if (result is null)
{
throw new InvalidOperationException("EntryTrace response payload was empty.");
}
var result = await response.Content.ReadFromJsonAsync<EntryTraceResponseModel>(SerializerOptions, cancellationToken).ConfigureAwait(false);
if (result is null)
{
throw new InvalidOperationException("EntryTrace response payload was empty.");
}
return result;
}

View File

@@ -9,4 +9,5 @@ internal sealed record EntryTraceResponseModel(
string ImageDigest,
DateTimeOffset GeneratedAt,
EntryTraceGraph Graph,
IReadOnlyList<string> Ndjson);
IReadOnlyList<string> Ndjson,
EntryTracePlan? BestPlan);

View File

@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Text.Json.Serialization;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.WebService.Contracts;
public sealed record PolicyVexLookupRequest
{
[JsonPropertyName("advisory_keys")]
public IReadOnlyList<string> AdvisoryKeys { get; init; } = Array.Empty<string>();
[JsonPropertyName("purls")]
public IReadOnlyList<string> Purls { get; init; } = Array.Empty<string>();
[JsonPropertyName("providers")]
public IReadOnlyList<string> Providers { get; init; } = Array.Empty<string>();
[JsonPropertyName("statuses")]
public IReadOnlyList<string> Statuses { get; init; } = Array.Empty<string>();
[Range(1, 500)]
[JsonPropertyName("limit")]
public int Limit { get; init; } = 200;
}
public sealed record PolicyVexLookupResponse(
IReadOnlyList<PolicyVexLookupItem> Results,
int TotalStatements,
DateTimeOffset GeneratedAtUtc);
public sealed record PolicyVexLookupItem(
string AdvisoryKey,
IReadOnlyList<string> Aliases,
IReadOnlyList<PolicyVexStatement> Statements);
public sealed record PolicyVexStatement(
string ObservationId,
string ProviderId,
string Status,
string ProductKey,
string? Purl,
string? Cpe,
string? Version,
string? Justification,
string? Detail,
DateTimeOffset FirstSeen,
DateTimeOffset LastSeen,
VexSignatureMetadata? Signature,
IReadOnlyDictionary<string, string> Metadata);

View File

@@ -13,6 +13,7 @@ public sealed record VexLinksetListItem(
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("vulnerabilityId")] string VulnerabilityId,
[property: JsonPropertyName("productKey")] string ProductKey,
[property: JsonPropertyName("scope")] VexLinksetScope Scope,
[property: JsonPropertyName("providerIds")] IReadOnlyList<string> ProviderIds,
[property: JsonPropertyName("statuses")] IReadOnlyList<string> Statuses,
[property: JsonPropertyName("aliases")] IReadOnlyList<string> Aliases,
@@ -38,3 +39,11 @@ public sealed record VexLinksetObservationRef(
[property: JsonPropertyName("providerId")] string ProviderId,
[property: JsonPropertyName("status")] string Status,
[property: JsonPropertyName("confidence")] double? Confidence);
public sealed record VexLinksetScope(
[property: JsonPropertyName("productKey")] string ProductKey,
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("version")] string? Version,
[property: JsonPropertyName("purl")] string? Purl,
[property: JsonPropertyName("cpe")] string? Cpe,
[property: JsonPropertyName("identifiers")] IReadOnlyList<string> Identifiers);

View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Text.Json.Serialization;
@@ -7,11 +8,13 @@ using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core.Canonicalization;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
using StellaOps.Excititor.WebService.Telemetry;
using DomainVexProductScope = StellaOps.Excititor.Core.Observations.VexProductScope;
namespace StellaOps.Excititor.WebService.Endpoints;
@@ -267,11 +270,13 @@ public static class LinksetEndpoints
private static VexLinksetListItem ToListItem(VexLinkset linkset)
{
var scope = BuildScope(linkset.Scope);
return new VexLinksetListItem(
LinksetId: linkset.LinksetId,
Tenant: linkset.Tenant,
VulnerabilityId: linkset.VulnerabilityId,
ProductKey: linkset.ProductKey,
Scope: scope,
ProviderIds: linkset.ProviderIds.ToList(),
Statuses: linkset.Statuses.ToList(),
Aliases: Array.Empty<string>(), // Aliases are in observations, not linksets
@@ -289,11 +294,13 @@ public static class LinksetEndpoints
private static VexLinksetDetailResponse ToDetailResponse(VexLinkset linkset)
{
var scope = BuildScope(linkset.Scope);
return new VexLinksetDetailResponse(
LinksetId: linkset.LinksetId,
Tenant: linkset.Tenant,
VulnerabilityId: linkset.VulnerabilityId,
ProductKey: linkset.ProductKey,
Scope: scope,
ProviderIds: linkset.ProviderIds.ToList(),
Statuses: linkset.Statuses.ToList(),
Confidence: linkset.Confidence.ToString().ToLowerInvariant(),
@@ -343,6 +350,17 @@ public static class LinksetEndpoints
var raw = $"{timestamp:O}|{id}";
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(raw));
}
private static VexLinksetScope BuildScope(DomainVexProductScope scope)
{
return new VexLinksetScope(
ProductKey: scope.ProductKey,
Type: scope.Type,
Version: scope.Version,
Purl: scope.Purl,
Cpe: scope.Cpe,
Identifiers: scope.Identifiers.ToList());
}
}
// Detail response for single linkset
@@ -351,6 +369,7 @@ public sealed record VexLinksetDetailResponse(
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("vulnerabilityId")] string VulnerabilityId,
[property: JsonPropertyName("productKey")] string ProductKey,
[property: JsonPropertyName("scope")] VexLinksetScope Scope,
[property: JsonPropertyName("providerIds")] IReadOnlyList<string> ProviderIds,
[property: JsonPropertyName("statuses")] IReadOnlyList<string> Statuses,
[property: JsonPropertyName("confidence")] string Confidence,

View File

@@ -0,0 +1,204 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Canonicalization;
using StellaOps.Excititor.Core.Orchestration;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
namespace StellaOps.Excititor.WebService.Endpoints;
/// <summary>
/// Policy-facing VEX lookup endpoints (EXCITITOR-POLICY-20-001).
/// Aggregation-only: returns raw observations/statements without consensus or severity.
/// </summary>
public static class PolicyEndpoints
{
public static void MapPolicyEndpoints(this WebApplication app)
{
app.MapPost("/policy/v1/vex/lookup", LookupVexAsync)
.WithName("Policy_VexLookup")
.WithDescription("Batch VEX lookup by advisory_key and product (aggregation-only)");
}
private static async Task<IResult> LookupVexAsync(
HttpContext context,
[FromBody] PolicyVexLookupRequest request,
IOptions<VexMongoStorageOptions> storageOptions,
[FromServices] IVexClaimStore claimStore,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
// AuthN/Z
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out _, out var tenantError))
{
return tenantError!;
}
// Validate input
if ((request.AdvisoryKeys.Count == 0) && (request.Purls.Count == 0))
{
return Results.BadRequest(new { error = new { code = "ERR_REQUEST", message = "advisory_keys or purls must be provided" } });
}
var canonicalizer = new VexAdvisoryKeyCanonicalizer();
var productCanonicalizer = new VexProductKeyCanonicalizer();
var canonicalAdvisories = request.AdvisoryKeys
.Where(a => !string.IsNullOrWhiteSpace(a))
.Select(a => canonicalizer.Canonicalize(a.Trim()))
.ToList();
var canonicalProducts = request.Purls
.Where(p => !string.IsNullOrWhiteSpace(p))
.Select(p => productCanonicalizer.Canonicalize(p.Trim(), purl: p.Trim()))
.ToList();
// Map requested statuses/providers for filtering
var statusFilter = request.Statuses
.Select(s => Enum.TryParse<VexClaimStatus>(s, true, out var parsed) ? parsed : (VexClaimStatus?)null)
.Where(p => p.HasValue)
.Select(p => p!.Value)
.ToImmutableHashSet();
var providerFilter = request.Providers
.Where(p => !string.IsNullOrWhiteSpace(p))
.Select(p => p.Trim())
.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase);
var limit = Math.Clamp(request.Limit, 1, 500);
var now = timeProvider.GetUtcNow();
var results = new List<PolicyVexLookupItem>();
var totalStatements = 0;
// For each advisory key, fetch claims and filter by product/provider/status
foreach (var advisory in canonicalAdvisories)
{
var claims = await claimStore
.FindByVulnerabilityAsync(advisory.AdvisoryKey, limit, cancellationToken)
.ConfigureAwait(false);
var filtered = claims
.Where(claim => MatchesProvider(providerFilter, claim))
.Where(claim => MatchesStatus(statusFilter, claim))
.Where(claim => MatchesProduct(canonicalProducts, claim))
.OrderByDescending(claim => claim.LastSeen)
.ThenBy(claim => claim.ProviderId, StringComparer.Ordinal)
.ThenBy(claim => claim.Product.Key, StringComparer.Ordinal)
.Take(limit)
.ToList();
totalStatements += filtered.Count;
var statements = filtered.Select(MapStatement).ToList();
var aliases = advisory.Aliases.ToList();
if (!aliases.Contains(advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase))
{
aliases.Add(advisory.AdvisoryKey);
}
results.Add(new PolicyVexLookupItem(
advisory.AdvisoryKey,
aliases,
statements));
}
var response = new PolicyVexLookupResponse(results, totalStatements, now);
return Results.Ok(response);
}
private static bool MatchesProvider(ISet<string> providers, VexClaim claim)
=> providers.Count == 0 || providers.Contains(claim.ProviderId, StringComparer.OrdinalIgnoreCase);
private static bool MatchesStatus(ISet<VexClaimStatus> statuses, VexClaim claim)
=> statuses.Count == 0 || statuses.Contains(claim.Status);
private static bool MatchesProduct(IEnumerable<VexCanonicalProductKey> requestedProducts, VexClaim claim)
{
if (!requestedProducts.Any())
{
return true;
}
return requestedProducts.Any(product =>
string.Equals(product.ProductKey, claim.Product.Key, StringComparison.OrdinalIgnoreCase) ||
product.Links.Any(link => string.Equals(link.Identifier, claim.Product.Key, StringComparison.OrdinalIgnoreCase)) ||
(!string.IsNullOrWhiteSpace(product.Purl) && string.Equals(product.Purl, claim.Product.Purl, StringComparison.OrdinalIgnoreCase)));
}
private static PolicyVexStatement MapStatement(VexClaim claim)
{
var observationId = $"{claim.ProviderId}:{claim.Document.Digest}";
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["document_digest"] = claim.Document.Digest,
["document_uri"] = claim.Document.SourceUri.ToString()
};
if (!string.IsNullOrWhiteSpace(claim.Document.Revision))
{
metadata["document_revision"] = claim.Document.Revision!;
}
return new PolicyVexStatement(
ObservationId: observationId,
ProviderId: claim.ProviderId,
Status: claim.Status.ToString(),
ProductKey: claim.Product.Key,
Purl: claim.Product.Purl,
Cpe: claim.Product.Cpe,
Version: claim.Product.Version,
Justification: claim.Justification?.ToString(),
Detail: claim.Detail,
FirstSeen: claim.FirstSeen,
LastSeen: claim.LastSeen,
Signature: claim.Document.Signature,
Metadata: metadata);
}
private static bool TryResolveTenant(
HttpContext context,
VexMongoStorageOptions options,
out string tenant,
out IResult? problem)
{
problem = null;
tenant = string.Empty;
var headerTenant = context.Request.Headers["X-Stella-Tenant"].FirstOrDefault();
if (!string.IsNullOrWhiteSpace(headerTenant))
{
tenant = headerTenant.Trim().ToLowerInvariant();
}
else if (!string.IsNullOrWhiteSpace(options.DefaultTenant))
{
tenant = options.DefaultTenant.Trim().ToLowerInvariant();
}
else
{
problem = Results.BadRequest(new
{
error = new { code = "ERR_TENANT", message = "X-Stella-Tenant header is required" }
});
return false;
}
return true;
}
}

View File

@@ -2282,6 +2282,7 @@ MirrorRegistrationEndpoints.MapMirrorRegistrationEndpoints(app);
// Evidence and Attestation APIs (WEB-OBS-53-001, WEB-OBS-54-001)
EvidenceEndpoints.MapEvidenceEndpoints(app);
AttestationEndpoints.MapAttestationEndpoints(app);
PolicyEndpoints.MapPolicyEndpoints(app);
// Observation and Linkset APIs (EXCITITOR-LNM-21-201, EXCITITOR-LNM-21-202)
ObservationEndpoints.MapObservationEndpoints(app);

View File

@@ -7,38 +7,63 @@ namespace StellaOps.Excititor.Worker.Options;
/// </summary>
public sealed class VexWorkerOrchestratorOptions
{
/// <summary>
/// Whether orchestrator integration is enabled.
/// </summary>
/// <summary>Whether orchestrator integration is enabled.</summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Interval between heartbeat emissions during job execution.
/// </summary>
/// <summary>Base address of the Orchestrator WebService (e.g. "https://orch.local/").</summary>
public Uri? BaseAddress { get; set; }
/// <summary>Logical job type registered with Orchestrator.</summary>
public string JobType { get; set; } = "exc-vex-ingest";
/// <summary>Unique worker identifier presented to Orchestrator.</summary>
public string WorkerId { get; set; } = "excititor-worker";
/// <summary>Optional task runner identifier (e.g. host name or pod).</summary>
public string? TaskRunnerId { get; set; }
/// <summary>Tenant header name; defaults to Orchestrator default.</summary>
public string TenantHeader { get; set; } = "X-Tenant-Id";
/// <summary>Tenant value to present when claiming jobs.</summary>
public string DefaultTenant { get; set; } = "default";
/// <summary>API key header name for worker auth.</summary>
public string ApiKeyHeader { get; set; } = "X-Worker-Token";
/// <summary>Optional API key value.</summary>
public string? ApiKey { get; set; }
/// <summary>Optional bearer token value.</summary>
public string? BearerToken { get; set; }
/// <summary>Interval between heartbeat emissions during job execution.</summary>
public TimeSpan HeartbeatInterval { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Minimum heartbeat interval (safety floor).
/// </summary>
/// <summary>Minimum heartbeat interval (safety floor).</summary>
public TimeSpan MinHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Maximum heartbeat interval (safety cap).
/// </summary>
/// <summary>Maximum heartbeat interval (safety cap).</summary>
public TimeSpan MaxHeartbeatInterval { get; set; } = TimeSpan.FromMinutes(2);
/// <summary>
/// Enable verbose logging for heartbeat/artifact events.
/// </summary>
/// <summary>Lease duration requested when claiming jobs.</summary>
public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5);
/// <summary>Lease extension requested on each heartbeat.</summary>
public TimeSpan HeartbeatExtend { get; set; } = TimeSpan.FromMinutes(1);
/// <summary>HTTP request timeout when talking to Orchestrator.</summary>
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(15);
/// <summary>Enable verbose logging for heartbeat/artifact events.</summary>
public bool EnableVerboseLogging { get; set; }
/// <summary>
/// Maximum number of artifact hashes to retain in state.
/// </summary>
/// <summary>Maximum number of artifact hashes to retain in state.</summary>
public int MaxArtifactHashes { get; set; } = 1000;
/// <summary>
/// Default tenant for worker jobs when not specified.
/// </summary>
public string DefaultTenant { get; set; } = "default";
/// <summary>Emit progress events for artifacts while running.</summary>
public bool EmitProgressForArtifacts { get; set; } = true;
/// <summary>Fallback to local state only when orchestrator is unreachable.</summary>
public bool AllowLocalFallback { get; set; } = true;
}

View File

@@ -1,5 +1,11 @@
using System;
using System.Collections.Immutable;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -13,7 +19,7 @@ namespace StellaOps.Excititor.Worker.Orchestration;
/// <summary>
/// Default implementation of <see cref="IVexWorkerOrchestratorClient"/>.
/// Stores heartbeats and artifacts locally and emits them to the orchestrator registry when configured.
/// Stores heartbeats and artifacts locally and, when configured, mirrors them to the Orchestrator worker API.
/// </summary>
internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
{
@@ -21,37 +27,94 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
private readonly TimeProvider _timeProvider;
private readonly IOptions<VexWorkerOrchestratorOptions> _options;
private readonly ILogger<VexWorkerOrchestratorClient> _logger;
private readonly HttpClient? _httpClient;
private readonly JsonSerializerOptions _serializerOptions;
private VexWorkerCommand? _pendingCommand;
private long _commandSequence;
public VexWorkerOrchestratorClient(
IVexConnectorStateRepository stateRepository,
TimeProvider timeProvider,
IOptions<VexWorkerOrchestratorOptions> options,
ILogger<VexWorkerOrchestratorClient> logger)
ILogger<VexWorkerOrchestratorClient> logger,
HttpClient? httpClient = null)
{
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_httpClient = httpClient;
_serializerOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
}
public ValueTask<VexWorkerJobContext> StartJobAsync(
public async ValueTask<VexWorkerJobContext> StartJobAsync(
string tenant,
string connectorId,
string? checkpoint,
CancellationToken cancellationToken = default)
{
var runId = Guid.NewGuid();
var startedAt = _timeProvider.GetUtcNow();
var context = new VexWorkerJobContext(tenant, connectorId, runId, checkpoint, startedAt);
var fallbackContext = new VexWorkerJobContext(tenant, connectorId, Guid.NewGuid(), checkpoint, startedAt);
if (!CanUseRemote())
{
return fallbackContext;
}
var claimRequest = new ClaimRequest(
WorkerId: _options.Value.WorkerId,
TaskRunnerId: _options.Value.TaskRunnerId ?? Environment.MachineName,
JobType: ResolveJobType(connectorId),
LeaseSeconds: ResolveLeaseSeconds(),
IdempotencyKey: $"exc-{connectorId}-{startedAt.ToUnixTimeSeconds()}");
var response = await PostAsync("api/v1/orchestrator/worker/claim", tenant, claimRequest, cancellationToken).ConfigureAwait(false);
if (response is null)
{
return fallbackContext;
}
if (response.StatusCode == HttpStatusCode.NoContent)
{
_logger.LogInformation("Orchestrator had no jobs for {ConnectorId}; continuing with local execution.", connectorId);
return fallbackContext;
}
if (!response.IsSuccessStatusCode)
{
await HandleErrorResponseAsync("claim", response, connectorId, cancellationToken).ConfigureAwait(false);
return fallbackContext;
}
var claim = await DeserializeAsync<ClaimResponse>(response, cancellationToken).ConfigureAwait(false);
if (claim is null)
{
return fallbackContext;
}
_logger.LogInformation(
"Orchestrator job started: tenant={Tenant} connector={ConnectorId} runId={RunId} checkpoint={Checkpoint}",
"Orchestrator job claimed: tenant={Tenant} connector={ConnectorId} jobId={JobId} leaseUntil={LeaseUntil:O}",
tenant,
connectorId,
runId,
checkpoint ?? "(none)");
claim.JobId,
claim.LeaseUntil);
return ValueTask.FromResult(context);
return new VexWorkerJobContext(
tenant,
connectorId,
claim.JobId,
checkpoint,
startedAt,
orchestratorJobId: claim.JobId,
orchestratorLeaseId: claim.LeaseId,
leaseExpiresAt: claim.LeaseUntil,
jobType: claim.JobType,
correlationId: claim.CorrelationId,
orchestratorRunId: claim.RunId);
}
public async ValueTask SendHeartbeatAsync(
@@ -87,6 +150,8 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
heartbeat.Progress,
heartbeat.LastArtifactHash);
}
await SendRemoteHeartbeatAsync(context, heartbeat, cancellationToken).ConfigureAwait(false);
}
public async ValueTask RecordArtifactAsync(
@@ -106,7 +171,7 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
: state.DocumentDigests;
// Add artifact hash if not already tracked (cap to avoid unbounded growth)
const int maxDigests = 1000;
var maxDigests = Math.Max(1, _options.Value.MaxArtifactHashes);
if (!digests.Contains(artifact.Hash))
{
digests = digests.Length >= maxDigests
@@ -129,6 +194,8 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
artifact.Hash,
artifact.Kind,
artifact.ProviderId);
await SendRemoteProgressForArtifactAsync(context, artifact, cancellationToken).ConfigureAwait(false);
}
public async ValueTask CompleteJobAsync(
@@ -165,6 +232,8 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
result.DocumentsProcessed,
result.ClaimsGenerated,
duration);
await SendRemoteCompletionAsync(context, result, cancellationToken).ConfigureAwait(false);
}
public async ValueTask FailJobAsync(
@@ -202,6 +271,13 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
context.ConnectorId,
errorCode,
retryAfterSeconds);
await SendRemoteCompletionAsync(
context,
new VexWorkerJobResult(0, 0, state.LastCheckpoint, state.LastArtifactHash, now),
cancellationToken,
success: false,
failureReason: Truncate($"{errorCode}: {errorMessage}", 256)).ConfigureAwait(false);
}
public ValueTask FailJobAsync(
@@ -232,16 +308,13 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
{
ArgumentNullException.ThrowIfNull(context);
// In this local implementation, commands are not externally sourced.
// Return Continue to indicate normal processing should continue.
// A full orchestrator integration would poll a command queue here.
if (!_options.Value.Enabled)
{
return ValueTask.FromResult<VexWorkerCommand?>(null);
}
// No pending commands in local mode
return ValueTask.FromResult<VexWorkerCommand?>(null);
var command = Interlocked.Exchange(ref _pendingCommand, null);
return ValueTask.FromResult(command);
}
public ValueTask AcknowledgeCommandAsync(
@@ -256,7 +329,6 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
context.RunId,
commandSequence);
// In local mode, acknowledgment is a no-op
return ValueTask.CompletedTask;
}
@@ -314,6 +386,12 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
state.ResumeTokens.IsEmpty ? ImmutableDictionary<string, string>.Empty : state.ResumeTokens);
}
private bool CanUseRemote()
{
var opts = _options.Value;
return opts.Enabled && _httpClient is not null && opts.BaseAddress is not null;
}
private static string Truncate(string? value, int maxLength)
{
if (string.IsNullOrEmpty(value))
@@ -325,4 +403,276 @@ internal sealed class VexWorkerOrchestratorClient : IVexWorkerOrchestratorClient
? value
: value[..maxLength];
}
private int ResolveLeaseSeconds()
{
var seconds = (int)Math.Round(_options.Value.DefaultLeaseDuration.TotalSeconds);
return Math.Clamp(seconds, 30, 3600);
}
private int ResolveHeartbeatExtendSeconds()
{
var opts = _options.Value;
var seconds = (int)Math.Round(opts.HeartbeatExtend.TotalSeconds);
var min = (int)Math.Round(opts.MinHeartbeatInterval.TotalSeconds);
var max = (int)Math.Round(opts.MaxHeartbeatInterval.TotalSeconds);
return Math.Clamp(seconds, min, max);
}
private string ResolveJobType(string connectorId)
{
return string.IsNullOrWhiteSpace(_options.Value.JobType)
? $"exc-vex-{connectorId}"
: _options.Value.JobType;
}
private async ValueTask SendRemoteHeartbeatAsync(
VexWorkerJobContext context,
VexWorkerHeartbeat heartbeat,
CancellationToken cancellationToken)
{
if (!CanUseRemote() || context.OrchestratorJobId is null || context.OrchestratorLeaseId is null)
{
return;
}
var request = new HeartbeatRequest(
context.OrchestratorLeaseId.Value,
ResolveHeartbeatExtendSeconds(),
IdempotencyKey: $"hb-{context.RunId}-{context.Sequence}");
var response = await PostAsync(
$"api/v1/orchestrator/worker/jobs/{context.OrchestratorJobId}/heartbeat",
context.Tenant,
request,
cancellationToken).ConfigureAwait(false);
if (response is null)
{
return;
}
if (response.IsSuccessStatusCode)
{
var hb = await DeserializeAsync<HeartbeatResponse>(response, cancellationToken).ConfigureAwait(false);
if (hb?.LeaseUntil is not null)
{
context.UpdateLease(hb.LeaseUntil);
}
return;
}
await HandleErrorResponseAsync("heartbeat", response, context.ConnectorId, cancellationToken).ConfigureAwait(false);
}
private async ValueTask SendRemoteProgressForArtifactAsync(
VexWorkerJobContext context,
VexWorkerArtifact artifact,
CancellationToken cancellationToken)
{
if (!CanUseRemote() || !_options.Value.EmitProgressForArtifacts || context.OrchestratorJobId is null || context.OrchestratorLeaseId is null)
{
return;
}
var metadata = Serialize(new
{
artifact.Hash,
artifact.Kind,
artifact.ProviderId,
artifact.DocumentId,
artifact.CreatedAt
});
var request = new ProgressRequest(
context.OrchestratorLeaseId.Value,
ProgressPercent: null,
Message: $"artifact:{artifact.Kind}",
Metadata: metadata,
IdempotencyKey: $"artifact-{artifact.Hash}");
var response = await PostAsync(
$"api/v1/orchestrator/worker/jobs/{context.OrchestratorJobId}/progress",
context.Tenant,
request,
cancellationToken).ConfigureAwait(false);
if (response is not null && !response.IsSuccessStatusCode)
{
await HandleErrorResponseAsync("progress", response, context.ConnectorId, cancellationToken).ConfigureAwait(false);
}
}
private async ValueTask SendRemoteCompletionAsync(
VexWorkerJobContext context,
VexWorkerJobResult result,
CancellationToken cancellationToken,
bool success = true,
string? failureReason = null)
{
if (!CanUseRemote() || context.OrchestratorJobId is null || context.OrchestratorLeaseId is null)
{
return;
}
var request = new CompleteRequest(
context.OrchestratorLeaseId.Value,
success,
success ? null : failureReason,
Artifacts: Array.Empty<ArtifactInput>(),
ResultDigest: result.LastArtifactHash,
IdempotencyKey: $"complete-{context.RunId}-{context.Sequence}");
var response = await PostAsync(
$"api/v1/orchestrator/worker/jobs/{context.OrchestratorJobId}/complete",
context.Tenant,
request,
cancellationToken).ConfigureAwait(false);
if (response is not null && !response.IsSuccessStatusCode)
{
await HandleErrorResponseAsync("complete", response, context.ConnectorId, cancellationToken).ConfigureAwait(false);
}
}
private async Task<HttpResponseMessage?> PostAsync<TPayload>(
string path,
string tenant,
TPayload payload,
CancellationToken cancellationToken)
{
if (!CanUseRemote())
{
return null;
}
var request = new HttpRequestMessage(HttpMethod.Post, path)
{
Content = new StringContent(JsonSerializer.Serialize(payload, _serializerOptions), Encoding.UTF8, "application/json")
};
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var opts = _options.Value;
request.Headers.TryAddWithoutValidation(string.IsNullOrWhiteSpace(opts.TenantHeader) ? "X-Tenant-Id" : opts.TenantHeader, tenant);
if (!string.IsNullOrWhiteSpace(opts.ApiKey))
{
request.Headers.TryAddWithoutValidation(string.IsNullOrWhiteSpace(opts.ApiKeyHeader) ? "X-Worker-Token" : opts.ApiKeyHeader, opts.ApiKey);
}
if (!string.IsNullOrWhiteSpace(opts.BearerToken))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", opts.BearerToken);
}
try
{
return await _httpClient!.SendAsync(request, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (opts.AllowLocalFallback)
{
_logger.LogWarning(ex, "Failed to contact Orchestrator ({Path}); continuing locally.", path);
StorePendingCommand(VexWorkerCommandKind.Retry, reason: "orchestrator_unreachable", retryAfterSeconds: 60);
return null;
}
}
private async ValueTask<T?> DeserializeAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken)
{
var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
return await JsonSerializer.DeserializeAsync<T>(stream, _serializerOptions, cancellationToken).ConfigureAwait(false);
}
private async Task HandleErrorResponseAsync(
string stage,
HttpResponseMessage response,
string connectorId,
CancellationToken cancellationToken)
{
ErrorResponse? error = null;
try
{
error = await DeserializeAsync<ErrorResponse>(response, cancellationToken).ConfigureAwait(false);
}
catch
{
// ignore parse issues; fall back to status code handling
}
var retryAfter = error?.RetryAfterSeconds;
switch (response.StatusCode)
{
case HttpStatusCode.TooManyRequests:
StorePendingCommand(VexWorkerCommandKind.Throttle, reason: error?.Message ?? "rate_limited", retryAfterSeconds: retryAfter ?? 60);
break;
case HttpStatusCode.Conflict:
StorePendingCommand(VexWorkerCommandKind.Retry, reason: error?.Message ?? "lease_conflict", retryAfterSeconds: retryAfter ?? 30);
break;
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.BadGateway:
case HttpStatusCode.GatewayTimeout:
StorePendingCommand(VexWorkerCommandKind.Pause, reason: error?.Message ?? "orchestrator_unavailable", retryAfterSeconds: retryAfter ?? 120);
break;
default:
StorePendingCommand(VexWorkerCommandKind.Retry, reason: error?.Message ?? "orchestrator_error", retryAfterSeconds: retryAfter ?? 30);
break;
}
_logger.LogWarning(
"Orchestrator {Stage} call failed for connector {ConnectorId}: {Status} {Error}",
stage,
connectorId,
response.StatusCode,
error?.Message ?? response.ReasonPhrase);
}
private void StorePendingCommand(VexWorkerCommandKind kind, string? reason = null, int? retryAfterSeconds = null)
{
var issuedAt = _timeProvider.GetUtcNow();
var sequence = Interlocked.Increment(ref _commandSequence);
var expiresAt = retryAfterSeconds.HasValue ? issuedAt.AddSeconds(retryAfterSeconds.Value) : (DateTimeOffset?)null;
_pendingCommand = new VexWorkerCommand(
kind,
sequence,
issuedAt,
expiresAt,
Throttle: kind == VexWorkerCommandKind.Throttle && retryAfterSeconds.HasValue
? new VexWorkerThrottleParams(null, null, retryAfterSeconds)
: null,
Reason: reason);
}
private string Serialize(object value) => JsonSerializer.Serialize(value, _serializerOptions);
private sealed record ClaimRequest(string WorkerId, string? TaskRunnerId, string? JobType, int? LeaseSeconds, string? IdempotencyKey);
private sealed record ClaimResponse(
Guid JobId,
Guid LeaseId,
string JobType,
string Payload,
string PayloadDigest,
int Attempt,
int MaxAttempts,
DateTimeOffset LeaseUntil,
string IdempotencyKey,
string? CorrelationId,
Guid? RunId,
string? ProjectId);
private sealed record HeartbeatRequest(Guid LeaseId, int? ExtendSeconds, string? IdempotencyKey);
private sealed record HeartbeatResponse(Guid JobId, Guid LeaseId, DateTimeOffset LeaseUntil, bool Acknowledged);
private sealed record ProgressRequest(Guid LeaseId, double? ProgressPercent, string? Message, string? Metadata, string? IdempotencyKey);
private sealed record CompleteRequest(Guid LeaseId, bool Success, string? Reason, IReadOnlyList<ArtifactInput>? Artifacts, string? ResultDigest, string? IdempotencyKey);
private sealed record ArtifactInput(string ArtifactType, string Uri, string Digest, string? MimeType, long? SizeBytes, string? Metadata);
private sealed record ErrorResponse(string Error, string Message, Guid? JobId, int? RetryAfterSeconds);
}

View File

@@ -109,7 +109,16 @@ services.AddSingleton<PluginCatalog>(provider =>
services.AddOptions<VexWorkerOrchestratorOptions>()
.Bind(configuration.GetSection("Excititor:Worker:Orchestrator"))
.ValidateOnStart();
services.AddSingleton<IVexWorkerOrchestratorClient, VexWorkerOrchestratorClient>();
services.AddHttpClient<IVexWorkerOrchestratorClient, VexWorkerOrchestratorClient>((provider, client) =>
{
var opts = provider.GetRequiredService<IOptions<VexWorkerOrchestratorOptions>>().Value;
if (opts.BaseAddress is not null)
{
client.BaseAddress = opts.BaseAddress;
}
client.Timeout = opts.RequestTimeout;
});
services.AddSingleton<VexWorkerHeartbeatService>();
services.AddSingleton<IVexProviderRunner, DefaultVexProviderRunner>();

View File

@@ -16,6 +16,7 @@ public sealed record VexLinkset
string tenant,
string vulnerabilityId,
string productKey,
VexProductScope scope,
IEnumerable<VexLinksetObservationRefModel> observations,
IEnumerable<VexObservationDisagreement>? disagreements = null,
DateTimeOffset? createdAt = null,
@@ -25,6 +26,7 @@ public sealed record VexLinkset
Tenant = VexObservation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant();
VulnerabilityId = VexObservation.EnsureNotNullOrWhiteSpace(vulnerabilityId, nameof(vulnerabilityId));
ProductKey = VexObservation.EnsureNotNullOrWhiteSpace(productKey, nameof(productKey));
Scope = scope ?? throw new ArgumentNullException(nameof(scope));
Observations = NormalizeObservations(observations);
Disagreements = NormalizeDisagreements(disagreements);
CreatedAt = (createdAt ?? DateTimeOffset.UtcNow).ToUniversalTime();
@@ -52,6 +54,11 @@ public sealed record VexLinkset
/// </summary>
public string ProductKey { get; }
/// <summary>
/// Canonical scope metadata for the product key.
/// </summary>
public VexProductScope Scope { get; }
/// <summary>
/// References to observations that contribute to this linkset.
/// </summary>
@@ -154,6 +161,7 @@ public sealed record VexLinkset
Tenant,
VulnerabilityId,
ProductKey,
Scope,
observations,
disagreements,
CreatedAt,

View File

@@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Excititor.Core.Canonicalization;
namespace StellaOps.Excititor.Core.Observations;
@@ -49,6 +50,7 @@ public sealed class VexLinksetExtractionService
foreach (var group in groups)
{
var linksetId = BuildLinksetId(group.Key.VulnerabilityId, group.Key.ProductKey);
var scope = BuildScope(group.Key.ProductKey);
var obsForGroup = group.Select(x => x.obs);
var evt = VexLinksetUpdatedEventFactory.Create(
@@ -56,6 +58,7 @@ public sealed class VexLinksetExtractionService
linksetId,
group.Key.VulnerabilityId,
group.Key.ProductKey,
scope,
obsForGroup,
disagreements ?? Enumerable.Empty<VexObservationDisagreement>(),
now);
@@ -69,5 +72,46 @@ public sealed class VexLinksetExtractionService
private static string BuildLinksetId(string vulnerabilityId, string productKey)
=> $"vex:{vulnerabilityId}:{productKey}".ToLowerInvariant();
private static VexProductScope BuildScope(string productKey)
{
var canonicalizer = new VexProductKeyCanonicalizer();
try
{
var canonical = canonicalizer.Canonicalize(productKey);
var identifiers = canonical.Links
.Where(link => link is not null && !string.IsNullOrWhiteSpace(link.Identifier))
.Select(link => link.Identifier.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
var purl = canonical.Links.FirstOrDefault(link => string.Equals(link.Type, "purl", StringComparison.OrdinalIgnoreCase))?.Identifier;
var cpe = canonical.Links.FirstOrDefault(link => string.Equals(link.Type, "cpe", StringComparison.OrdinalIgnoreCase))?.Identifier;
var version = ExtractVersion(purl ?? canonical.ProductKey);
return new VexProductScope(
ProductKey: canonical.ProductKey,
Type: canonical.Scope.ToString().ToLowerInvariant(),
Version: version,
Purl: purl,
Cpe: cpe,
Identifiers: identifiers);
}
catch
{
return VexProductScope.Unknown(productKey);
}
}
private static string? ExtractVersion(string? key)
{
if (string.IsNullOrWhiteSpace(key))
{
return null;
}
var at = key.LastIndexOf('@');
return at >= 0 && at < key.Length - 1 ? key[(at + 1)..] : null;
}
private static string Normalize(string value) => VexObservation.EnsureNotNullOrWhiteSpace(value, nameof(value));
}

View File

@@ -21,6 +21,7 @@ public static class VexLinksetUpdatedEventFactory
string linksetId,
string vulnerabilityId,
string productKey,
VexProductScope scope,
IEnumerable<VexObservation> observations,
IEnumerable<VexObservationDisagreement> disagreements,
DateTimeOffset createdAtUtc)
@@ -62,6 +63,7 @@ public static class VexLinksetUpdatedEventFactory
normalizedLinksetId,
normalizedVulnerabilityId,
normalizedProductKey,
scope,
observationRefs,
disagreementList,
createdAtUtc);
@@ -151,6 +153,7 @@ public sealed record VexLinksetUpdatedEvent(
string LinksetId,
string VulnerabilityId,
string ProductKey,
VexProductScope Scope,
ImmutableArray<VexLinksetObservationRefCore> Observations,
ImmutableArray<VexObservationDisagreement> Disagreements,
DateTimeOffset CreatedAtUtc);

View File

@@ -0,0 +1,18 @@
using System.Collections.Immutable;
namespace StellaOps.Excititor.Core.Observations;
/// <summary>
/// Canonical scope metadata derived from a product identifier.
/// </summary>
public sealed record VexProductScope(
string ProductKey,
string Type,
string? Version,
string? Purl,
string? Cpe,
ImmutableArray<string> Identifiers)
{
public static VexProductScope Unknown(string productKey) =>
new(productKey ?? string.Empty, "unknown", null, null, null, ImmutableArray<string>.Empty);
}

View File

@@ -103,13 +103,25 @@ public sealed record VexWorkerJobContext
string connectorId,
Guid runId,
string? checkpoint,
DateTimeOffset startedAt)
DateTimeOffset startedAt,
Guid? orchestratorJobId = null,
Guid? orchestratorLeaseId = null,
DateTimeOffset? leaseExpiresAt = null,
string? jobType = null,
string? correlationId = null,
Guid? orchestratorRunId = null)
{
Tenant = EnsureNotNullOrWhiteSpace(tenant, nameof(tenant));
ConnectorId = EnsureNotNullOrWhiteSpace(connectorId, nameof(connectorId));
RunId = runId;
Checkpoint = checkpoint?.Trim();
StartedAt = startedAt;
OrchestratorJobId = orchestratorJobId;
OrchestratorLeaseId = orchestratorLeaseId;
LeaseExpiresAt = leaseExpiresAt;
JobType = jobType;
CorrelationId = correlationId;
OrchestratorRunId = orchestratorRunId;
}
public string Tenant { get; }
@@ -117,6 +129,12 @@ public sealed record VexWorkerJobContext
public Guid RunId { get; }
public string? Checkpoint { get; }
public DateTimeOffset StartedAt { get; }
public Guid? OrchestratorJobId { get; }
public Guid? OrchestratorLeaseId { get; }
public DateTimeOffset? LeaseExpiresAt { get; private set; }
public string? JobType { get; }
public string? CorrelationId { get; }
public Guid? OrchestratorRunId { get; }
/// <summary>
/// Current sequence number for heartbeats.
@@ -128,6 +146,11 @@ public sealed record VexWorkerJobContext
/// </summary>
public long NextSequence() => ++Sequence;
/// <summary>
/// Updates the tracked lease expiration when the orchestrator extends it.
/// </summary>
public void UpdateLease(DateTimeOffset leaseUntil) => LeaseExpiresAt = leaseUntil;
private static string EnsureNotNullOrWhiteSpace(string value, string name)
=> string.IsNullOrWhiteSpace(value) ? throw new ArgumentException($"{name} must be provided.", name) : value.Trim();
}

View File

@@ -57,6 +57,15 @@ internal sealed class MongoVexLinksetEventPublisher : IVexLinksetEventPublisher
LinksetId = @event.LinksetId,
VulnerabilityId = @event.VulnerabilityId,
ProductKey = @event.ProductKey,
Scope = new VexLinksetScopeRecord
{
ProductKey = @event.Scope.ProductKey,
Type = @event.Scope.Type,
Version = @event.Scope.Version,
Purl = @event.Scope.Purl,
Cpe = @event.Scope.Cpe,
Identifiers = @event.Scope.Identifiers.ToList(),
},
Observations = @event.Observations
.Select(o => new VexLinksetEventObservationRecord
{

View File

@@ -98,6 +98,7 @@ internal sealed class MongoVexLinksetStore : IVexLinksetStore
normalizedTenant,
normalizedVuln,
normalizedProduct,
scope: VexProductScope.Unknown(normalizedProduct),
observations: Array.Empty<VexLinksetObservationRefModel>(),
disagreements: null,
createdAt: DateTimeOffset.UtcNow,
@@ -275,6 +276,7 @@ internal sealed class MongoVexLinksetStore : IVexLinksetStore
LinksetId = linkset.LinksetId,
VulnerabilityId = linkset.VulnerabilityId.ToLowerInvariant(),
ProductKey = linkset.ProductKey.ToLowerInvariant(),
Scope = ToScopeRecord(linkset.Scope),
ProviderIds = linkset.ProviderIds.ToList(),
Statuses = linkset.Statuses.ToList(),
CreatedAt = linkset.CreatedAt.UtcDateTime,
@@ -326,14 +328,49 @@ internal sealed class MongoVexLinksetStore : IVexLinksetStore
d.Confidence))
.ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
var scope = record.Scope is not null
? ToScope(record.Scope)
: VexProductScope.Unknown(record.ProductKey);
return new VexLinkset(
linksetId: record.LinksetId,
tenant: record.Tenant,
vulnerabilityId: record.VulnerabilityId,
productKey: record.ProductKey,
scope: scope,
observations: observations,
disagreements: disagreements,
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
updatedAt: new DateTimeOffset(record.UpdatedAt, TimeSpan.Zero));
}
private static VexLinksetScopeRecord ToScopeRecord(VexProductScope scope)
{
return new VexLinksetScopeRecord
{
ProductKey = scope.ProductKey,
Type = scope.Type,
Version = scope.Version,
Purl = scope.Purl,
Cpe = scope.Cpe,
Identifiers = scope.Identifiers.ToList()
};
}
private static VexProductScope ToScope(VexLinksetScopeRecord record)
{
var identifiers = record.Identifiers?
.Where(id => !string.IsNullOrWhiteSpace(id))
.Select(id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToImmutableArray() ?? ImmutableArray<string>.Empty;
return new VexProductScope(
ProductKey: record.ProductKey ?? string.Empty,
Type: record.Type ?? "unknown",
Version: record.Version,
Purl: record.Purl,
Cpe: record.Cpe,
Identifiers: identifiers);
}
}

View File

@@ -484,6 +484,8 @@ internal sealed class VexLinksetRecord
public string ProductKey { get; set; } = default!;
public VexLinksetScopeRecord? Scope { get; set; }
public List<string> ProviderIds { get; set; } = new();
public List<string> Statuses { get; set; } = new();
@@ -1397,6 +1399,8 @@ internal sealed class VexLinksetEventRecord
public string ProductKey { get; set; } = default!;
public VexLinksetScopeRecord? Scope { get; set; }
public List<VexLinksetEventObservationRecord> Observations { get; set; } = new();
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
@@ -1410,6 +1414,24 @@ internal sealed class VexLinksetEventRecord
public int ObservationCount { get; set; } = 0;
}
internal sealed class VexLinksetScopeRecord
{
public string ProductKey { get; set; } = string.Empty;
public string Type { get; set; } = string.Empty;
public string? Version { get; set; }
= null;
public string? Purl { get; set; }
= null;
public string? Cpe { get; set; }
= null;
public List<string> Identifiers { get; set; } = new();
}
[BsonIgnoreExtraElements]
internal sealed class VexLinksetEventObservationRecord
{

View File

@@ -32,6 +32,13 @@ public sealed class VexLinksetUpdatedEventFactoryTests
linksetId: "link-123",
vulnerabilityId: "CVE-2025-0001",
productKey: "pkg:demo/app",
scope: new VexProductScope(
ProductKey: "pkg:demo/app",
Type: "package",
Version: "1.0.0",
Purl: "pkg:demo/app@1.0.0",
Cpe: null,
Identifiers: ImmutableArray.Create("pkg:demo/app@1.0.0")),
observations,
disagreements,
now);

View File

@@ -0,0 +1,97 @@
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Contracts;
namespace StellaOps.Excititor.WebService.Tests;
public sealed class PolicyEndpointsTests
{
[Fact]
public async Task VexLookup_ReturnsStatements_ForAdvisoryAndPurl()
{
var claims = CreateSampleClaims();
using var factory = new TestWebApplicationFactory(
configureServices: services =>
{
services.RemoveAll<IVexClaimStore>();
services.AddSingleton<IVexClaimStore>(new StubClaimStore(claims));
services.AddTestAuthentication();
});
using var client = factory.CreateClient(new() { AllowAutoRedirect = false });
client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "vex.read");
client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test");
var request = new PolicyVexLookupRequest
{
AdvisoryKeys = new[] { "CVE-2025-1234" },
Purls = new[] { "pkg:maven/org.example/app@1.2.3" },
Limit = 10
};
var response = await client.PostAsJsonAsync("/policy/v1/vex/lookup", request);
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadFromJsonAsync<PolicyVexLookupResponse>();
Assert.NotNull(body);
Assert.Single(body!.Results);
var result = body.Results.First();
Assert.Equal("CVE-2025-1234", result.AdvisoryKey);
Assert.Single(result.Statements);
var statement = result.Statements.First();
Assert.Equal("pkg:maven/org.example/app@1.2.3", statement.Purl);
Assert.Equal("affected", statement.Status.ToLowerInvariant());
}
private static IReadOnlyList<VexClaim> CreateSampleClaims()
{
var now = DateTimeOffset.Parse("2025-12-01T12:00:00Z");
var product = new VexProduct(
key: "pkg:maven/org.example/app",
name: "Example App",
version: "1.2.3",
purl: "pkg:maven/org.example/app@1.2.3");
var document = new VexClaimDocument(
format: VexDocumentFormat.Csaf,
digest: "sha256:deadbeef",
sourceUri: new Uri("https://example.org/advisory.json"),
revision: "v1",
signature: null);
var claim = new VexClaim(
vulnerabilityId: "CVE-2025-1234",
providerId: "ghsa",
product: product,
status: VexClaimStatus.Affected,
document: document,
firstSeen: now.AddHours(-1),
lastSeen: now);
return new[] { claim };
}
private sealed class StubClaimStore : IVexClaimStore
{
private readonly IReadOnlyList<VexClaim> _claims;
public StubClaimStore(IReadOnlyList<VexClaim> claims)
{
_claims = claims;
}
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
=> ValueTask.CompletedTask;
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(_claims.Where(c => c.VulnerabilityId == vulnerabilityId && c.Product.Key == productKey).ToList());
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(_claims.Where(c => c.VulnerabilityId == vulnerabilityId).Take(limit).ToList());
}
}

View File

@@ -41,5 +41,6 @@
<Compile Include="GraphTooltipFactoryTests.cs" />
<Compile Include="AttestationVerifyEndpointTests.cs" />
<Compile Include="OpenApiDiscoveryEndpointTests.cs" />
<Compile Include="PolicyEndpointsTests.cs" />
</ItemGroup>
</Project>

View File

@@ -212,6 +212,81 @@ internal static class TestServiceOverrides
_records.Add(record);
return Task.CompletedTask;
}
public Task<AirgapImportRecord?> FindByBundleIdAsync(
string tenantId,
string bundleId,
string? mirrorGeneration,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var matches = _records
.Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase)
&& string.Equals(r.BundleId, bundleId, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(mirrorGeneration))
{
matches = matches.Where(r => string.Equals(r.MirrorGeneration, mirrorGeneration, StringComparison.OrdinalIgnoreCase));
}
var result = matches
.OrderByDescending(r => r.MirrorGeneration, StringComparer.OrdinalIgnoreCase)
.FirstOrDefault();
return Task.FromResult(result);
}
public Task<IReadOnlyList<AirgapImportRecord>> ListAsync(
string tenantId,
string? publisherFilter,
DateTimeOffset? importedAfter,
int limit,
int offset,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var query = _records
.Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(publisherFilter))
{
query = query.Where(r => string.Equals(r.Publisher, publisherFilter, StringComparison.OrdinalIgnoreCase));
}
if (importedAfter.HasValue)
{
query = query.Where(r => r.ImportedAt > importedAfter.Value);
}
var result = query
.OrderByDescending(r => r.ImportedAt)
.Skip(Math.Max(0, offset))
.Take(Math.Clamp(limit, 1, 1000))
.ToList()
.AsReadOnly();
return Task.FromResult((IReadOnlyList<AirgapImportRecord>)result);
}
public Task<int> CountAsync(
string tenantId,
string? publisherFilter,
DateTimeOffset? importedAfter,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var count = _records
.Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase))
.Where(r => string.IsNullOrWhiteSpace(publisherFilter) ||
string.Equals(r.Publisher, publisherFilter, StringComparison.OrdinalIgnoreCase))
.Where(r => !importedAfter.HasValue || r.ImportedAt > importedAfter.Value)
.Count();
return Task.FromResult(count);
}
}
private sealed class StubIngestOrchestrator : IVexIngestOrchestrator

View File

@@ -323,14 +323,17 @@ public sealed class DefaultVexProviderRunnerIntegrationTests : IAsyncLifetime
}
}
private sealed class NoopClaimStore : IVexClaimStore
{
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.CompletedTask;
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
}
private sealed class NoopClaimStore : IVexClaimStore
{
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.CompletedTask;
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
}
private sealed class NoopNormalizerRouter : IVexNormalizerRouter
{

View File

@@ -484,14 +484,17 @@ public sealed class DefaultVexProviderRunnerTests
=> ValueTask.FromResult<VexRawDocument?>(null);
}
private sealed class NoopClaimStore : IVexClaimStore
{
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.CompletedTask;
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
}
private sealed class NoopClaimStore : IVexClaimStore
{
public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.CompletedTask;
public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
}
private sealed class NoopProviderStore : IVexProviderStore
{

View File

@@ -2,6 +2,10 @@ using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
@@ -18,11 +22,6 @@ public class VexWorkerOrchestratorClientTests
{
private readonly InMemoryConnectorStateRepository _stateRepository = new();
private readonly FakeTimeProvider _timeProvider = new();
private readonly IOptions<VexWorkerOrchestratorOptions> _options = Microsoft.Extensions.Options.Options.Create(new VexWorkerOrchestratorOptions
{
Enabled = true,
DefaultTenant = "test-tenant"
});
[Fact]
public async Task StartJobAsync_CreatesJobContext()
@@ -61,6 +60,172 @@ public class VexWorkerOrchestratorClientTests
Assert.NotNull(state.LastHeartbeatAt);
}
[Fact]
public async Task StartJobAsync_UsesOrchestratorClaim_WhenAvailable()
{
var jobId = Guid.NewGuid();
var leaseId = Guid.NewGuid();
var handler = new StubHandler(request =>
{
if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
{
return JsonResponse(new
{
jobId,
leaseId,
jobType = "exc-vex-ingest",
payload = "{}",
payloadDigest = "sha256:abc",
attempt = 1,
maxAttempts = 3,
leaseUntil = "2025-12-01T12:00:00Z",
idempotencyKey = "abc",
correlationId = "corr-1",
runId = (Guid?)null,
projectId = (string?)null
});
}
return new HttpResponseMessage(HttpStatusCode.NotFound);
});
var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
var client = CreateClient(httpClient, opts => opts.BaseAddress = httpClient.BaseAddress);
var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
Assert.Equal(jobId, context.RunId);
Assert.Equal(jobId, context.OrchestratorJobId);
Assert.Equal(leaseId, context.OrchestratorLeaseId);
Assert.Contains(handler.Requests, r => r.RequestUri!.AbsolutePath.EndsWith("/claim"));
}
[Fact]
public async Task SendHeartbeatAsync_ExtendsLeaseViaOrchestrator()
{
var jobId = Guid.NewGuid();
var leaseId = Guid.NewGuid();
var leaseUntil = DateTimeOffset.Parse("2025-12-01T12:05:00Z");
var handler = new StubHandler(request =>
{
if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
{
return JsonResponse(new
{
jobId,
leaseId,
jobType = "exc-vex-ingest",
payload = "{}",
payloadDigest = "sha256:abc",
attempt = 1,
maxAttempts = 3,
leaseUntil = "2025-12-01T12:00:00Z",
idempotencyKey = "abc",
correlationId = "corr-1",
runId = (Guid?)null,
projectId = (string?)null
});
}
if (request.RequestUri!.AbsolutePath.Contains("/heartbeat"))
{
return JsonResponse(new
{
jobId,
leaseId,
leaseUntil,
acknowledged = true
});
}
return new HttpResponseMessage(HttpStatusCode.NotFound);
});
var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
var client = CreateClient(httpClient, opts =>
{
opts.BaseAddress = httpClient.BaseAddress;
opts.HeartbeatExtend = TimeSpan.FromSeconds(45);
});
var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
var heartbeat = new VexWorkerHeartbeat(
VexWorkerHeartbeatStatus.Running,
Progress: 10,
QueueDepth: null,
LastArtifactHash: "sha256:abc",
LastArtifactKind: "vex-document",
ErrorCode: null,
RetryAfterSeconds: null);
await client.SendHeartbeatAsync(context, heartbeat);
Assert.Equal(leaseUntil, context.LeaseExpiresAt);
Assert.Contains(handler.Requests, r => r.RequestUri!.AbsolutePath.Contains("/heartbeat"));
}
[Fact]
public async Task SendHeartbeatAsync_StoresThrottleCommand_On429()
{
var jobId = Guid.NewGuid();
var leaseId = Guid.NewGuid();
var handler = new StubHandler(request =>
{
if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
{
return JsonResponse(new
{
jobId,
leaseId,
jobType = "exc-vex-ingest",
payload = "{}",
payloadDigest = "sha256:abc",
attempt = 1,
maxAttempts = 3,
leaseUntil = "2025-12-01T12:00:00Z",
idempotencyKey = "abc",
correlationId = "corr-1",
runId = (Guid?)null,
projectId = (string?)null
});
}
if (request.RequestUri!.AbsolutePath.Contains("/heartbeat"))
{
var error = new { error = "rate_limited", message = "slow down", jobId, retryAfterSeconds = 15 };
return JsonResponse(error, HttpStatusCode.TooManyRequests);
}
return new HttpResponseMessage(HttpStatusCode.NotFound);
});
var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
var client = CreateClient(httpClient, opts => opts.BaseAddress = httpClient.BaseAddress);
var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
var heartbeat = new VexWorkerHeartbeat(
VexWorkerHeartbeatStatus.Running,
Progress: 5,
QueueDepth: null,
LastArtifactHash: null,
LastArtifactKind: null,
ErrorCode: null,
RetryAfterSeconds: null);
await client.SendHeartbeatAsync(context, heartbeat);
var command = await client.GetPendingCommandAsync(context);
Assert.NotNull(command);
Assert.Equal(VexWorkerCommandKind.Throttle, command!.Kind);
Assert.Equal(15, command.Throttle?.CooldownSeconds);
}
[Fact]
public async Task RecordArtifactAsync_TracksArtifactHash()
{
@@ -140,12 +305,25 @@ public class VexWorkerOrchestratorClientTests
Assert.Equal(3, context.NextSequence());
}
private VexWorkerOrchestratorClient CreateClient()
=> new(
private VexWorkerOrchestratorClient CreateClient(
HttpClient? httpClient = null,
Action<VexWorkerOrchestratorOptions>? configure = null)
{
var opts = new VexWorkerOrchestratorOptions
{
Enabled = true,
DefaultTenant = "test-tenant"
};
configure?.Invoke(opts);
return new VexWorkerOrchestratorClient(
_stateRepository,
_timeProvider,
_options,
NullLogger<VexWorkerOrchestratorClient>.Instance);
Microsoft.Extensions.Options.Options.Create(opts),
NullLogger<VexWorkerOrchestratorClient>.Instance,
httpClient);
}
private sealed class FakeTimeProvider : TimeProvider
{
@@ -175,4 +353,31 @@ public class VexWorkerOrchestratorClientTests
public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
=> ValueTask.FromResult<IReadOnlyCollection<VexConnectorState>>(_states.Values.ToList());
}
private sealed class StubHandler : HttpMessageHandler
{
private readonly Func<HttpRequestMessage, HttpResponseMessage> _responder;
public StubHandler(Func<HttpRequestMessage, HttpResponseMessage> responder)
{
_responder = responder;
}
public List<HttpRequestMessage> Requests { get; } = new();
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
Requests.Add(request);
return Task.FromResult(_responder(request));
}
}
private static HttpResponseMessage JsonResponse(object payload, HttpStatusCode status = HttpStatusCode.OK)
{
var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase });
return new HttpResponseMessage(status)
{
Content = new StringContent(json, Encoding.UTF8, "application/json")
};
}
}

View File

@@ -0,0 +1,239 @@
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Policy;
using StellaOps.Policy.Engine.Caching;
using StellaOps.Policy.Engine.Evaluation;
using StellaOps.Policy.Engine.Services;
using StellaOps.PolicyDsl;
namespace StellaOps.Policy.Engine.BatchEvaluation;
internal sealed record BatchEvaluationRequestDto(
string TenantId,
IReadOnlyList<BatchEvaluationItemDto> Items,
int? PageSize = null,
string? PageToken = null,
int? BudgetMs = null);
internal sealed record BatchEvaluationItemDto(
string PackId,
int Version,
string SubjectPurl,
string AdvisoryId,
EvaluationSeverityDto Severity,
AdvisoryDto Advisory,
VexEvidenceDto Vex,
SbomDto Sbom,
ExceptionsDto Exceptions,
ReachabilityDto Reachability,
DateTimeOffset? EvaluationTimestamp,
bool BypassCache = false);
internal sealed record EvaluationSeverityDto(string Normalized, decimal? Score = null);
internal sealed record AdvisoryDto(IDictionary<string, string> Metadata, string Source = "unknown");
internal sealed record VexEvidenceDto(IReadOnlyList<VexStatementDto> Statements);
internal sealed record VexStatementDto(string Status, string Justification, string StatementId, DateTimeOffset? Timestamp = null);
internal sealed record SbomDto(IReadOnlyList<string> Tags, IReadOnlyList<ComponentDto>? Components = null);
internal sealed record ComponentDto(
string Name,
string Version,
string Type,
string? Purl = null,
IDictionary<string, string>? Metadata = null);
internal sealed record ExceptionsDto(
IDictionary<string, PolicyExceptionEffect>? Effects = null,
IReadOnlyList<ExceptionInstanceDto>? Instances = null);
internal sealed record ExceptionInstanceDto(
string Id,
string EffectId,
ExceptionScopeDto Scope,
DateTimeOffset CreatedAt,
IDictionary<string, string>? Metadata = null);
internal sealed record ExceptionScopeDto(
IReadOnlyList<string>? RuleNames = null,
IReadOnlyList<string>? Severities = null,
IReadOnlyList<string>? Sources = null,
IReadOnlyList<string>? Tags = null);
internal sealed record ReachabilityDto(
string State,
decimal Confidence = 0m,
decimal Score = 0m,
bool HasRuntimeEvidence = false,
string? Source = null,
string? Method = null,
string? EvidenceRef = null);
internal sealed record BatchEvaluationResultDto(
string PackId,
int Version,
string PolicyDigest,
string Status,
string? Severity,
string? RuleName,
int? Priority,
IReadOnlyDictionary<string, string> Annotations,
IReadOnlyList<string> Warnings,
PolicyExceptionApplication? AppliedException,
string CorrelationId,
bool Cached,
CacheSource CacheSource,
long EvaluationDurationMs);
internal sealed record BatchEvaluationResponseDto(
IReadOnlyList<BatchEvaluationResultDto> Results,
string? NextPageToken,
int Total,
int Returned,
int CacheHits,
int CacheMisses,
long DurationMs,
long? BudgetRemainingMs);
internal static class BatchEvaluationValidator
{
public static bool TryValidate(BatchEvaluationRequestDto request, out string error)
{
if (request is null)
{
error = "Request body is required.";
return false;
}
if (request.Items is null || request.Items.Count == 0)
{
error = "At least one item is required.";
return false;
}
if (request.PageSize is int size && (size <= 0 || size > 500))
{
error = "PageSize must be between 1 and 500.";
return false;
}
if (request.Items.Any(static i => i.EvaluationTimestamp is null))
{
error = "Each item must provide evaluationTimestamp to keep evaluation deterministic.";
return false;
}
error = string.Empty;
return true;
}
}
internal static class BatchEvaluationMapper
{
public static IReadOnlyList<RuntimeEvaluationRequest> ToRuntimeRequests(string tenantId, IEnumerable<BatchEvaluationItemDto> items)
{
return items.Select(item => ToRuntimeRequest(tenantId, item)).ToList();
}
private static RuntimeEvaluationRequest ToRuntimeRequest(string tenantId, BatchEvaluationItemDto item)
{
var severity = new PolicyEvaluationSeverity(
item.Severity.Normalized,
item.Severity.Score);
var advisory = new PolicyEvaluationAdvisory(
item.Advisory.Source,
item.Advisory.Metadata.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase));
var vex = new PolicyEvaluationVexEvidence(
item.Vex.Statements
.Select(stmt => new PolicyEvaluationVexStatement(
stmt.Status,
stmt.Justification,
stmt.StatementId,
stmt.Timestamp))
.ToImmutableArray());
var sbom = new PolicyEvaluationSbom(
item.Sbom.Tags.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase),
(item.Sbom.Components ?? Array.Empty<ComponentDto>())
.Select(comp => new PolicyEvaluationComponent(
comp.Name,
comp.Version,
comp.Type,
comp.Purl,
(comp.Metadata ?? new Dictionary<string, string>())
.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase)))
.ToImmutableArray());
var exceptions = new PolicyEvaluationExceptions(
(item.Exceptions.Effects ?? new Dictionary<string, PolicyExceptionEffect>())
.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase),
(item.Exceptions.Instances ?? Array.Empty<ExceptionInstanceDto>())
.Select(instance => new PolicyEvaluationExceptionInstance(
instance.Id,
instance.EffectId,
new PolicyEvaluationExceptionScope(
Normalize(instance.Scope.RuleNames),
Normalize(instance.Scope.Severities),
Normalize(instance.Scope.Sources),
Normalize(instance.Scope.Tags)),
instance.CreatedAt,
(instance.Metadata ?? new Dictionary<string, string>())
.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase)))
.ToImmutableArray());
var reachability = new PolicyEvaluationReachability(
item.Reachability.State,
item.Reachability.Confidence,
item.Reachability.Score,
item.Reachability.HasRuntimeEvidence,
item.Reachability.Source,
item.Reachability.Method,
item.Reachability.EvidenceRef);
return new RuntimeEvaluationRequest(
PackId: item.PackId,
Version: item.Version,
TenantId: tenantId,
SubjectPurl: item.SubjectPurl,
AdvisoryId: item.AdvisoryId,
Severity: severity,
Advisory: advisory,
Vex: vex,
Sbom: sbom,
Exceptions: exceptions,
Reachability: reachability,
EvaluationTimestamp: item.EvaluationTimestamp,
BypassCache: item.BypassCache);
}
private static ImmutableHashSet<string> Normalize(IReadOnlyList<string>? values)
{
return (values ?? Array.Empty<string>())
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value.Trim())
.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase);
}
}
internal interface IRuntimeEvaluationExecutor
{
Task<RuntimeEvaluationResponse> EvaluateAsync(RuntimeEvaluationRequest request, CancellationToken cancellationToken);
}
internal sealed class RuntimeEvaluationExecutor : IRuntimeEvaluationExecutor
{
private readonly PolicyRuntimeEvaluationService _service;
public RuntimeEvaluationExecutor(PolicyRuntimeEvaluationService service)
{
_service = service ?? throw new ArgumentNullException(nameof(service));
}
public Task<RuntimeEvaluationResponse> EvaluateAsync(RuntimeEvaluationRequest request, CancellationToken cancellationToken) =>
_service.EvaluateAsync(request, cancellationToken);
}

View File

@@ -0,0 +1,147 @@
using System.Diagnostics;
using System.Linq;
using Microsoft.AspNetCore.Http.HttpResults;
using Microsoft.AspNetCore.Mvc;
using StellaOps.Auth.Abstractions;
using StellaOps.Policy.Engine.BatchEvaluation;
using StellaOps.Policy.Engine.Services;
namespace StellaOps.Policy.Engine.Endpoints;
internal static class BatchEvaluationEndpoint
{
public static IEndpointRouteBuilder MapBatchEvaluation(this IEndpointRouteBuilder routes)
{
var group = routes.MapGroup("/policy/eval")
.RequireAuthorization()
.WithTags("Policy Evaluation");
group.MapPost("/batch", EvaluateBatchAsync)
.WithName("PolicyEngine.BatchEvaluate")
.WithSummary("Batch-evaluate policy packs against advisory/VEX/SBOM tuples with deterministic ordering and cache-aware responses.")
.Produces<BatchEvaluationResponseDto>(StatusCodes.Status200OK)
.Produces<ProblemHttpResult>(StatusCodes.Status400BadRequest);
return routes;
}
private static async Task<IResult> EvaluateBatchAsync(
HttpContext httpContext,
[FromBody] BatchEvaluationRequestDto request,
IRuntimeEvaluationExecutor evaluator,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
var scopeResult = ScopeAuthorization.RequireScope(httpContext, StellaOpsScopes.PolicyRead);
if (scopeResult is not null)
{
return scopeResult;
}
if (!BatchEvaluationValidator.TryValidate(request, out var error))
{
return Results.BadRequest(new ProblemDetails
{
Title = "Invalid request",
Detail = error,
Status = StatusCodes.Status400BadRequest
});
}
if (!TryParseOffset(request.PageToken, out var offset))
{
return Results.BadRequest(new ProblemDetails
{
Title = "Invalid pageToken",
Detail = "pageToken must be a non-negative integer offset.",
Status = StatusCodes.Status400BadRequest
});
}
var pageSize = Math.Clamp(request.PageSize ?? 100, 1, 500);
var budgetMs = request.BudgetMs;
var sw = Stopwatch.StartNew();
var pageItems = request.Items
.Skip(offset)
.Take(pageSize)
.ToList();
var runtimeRequests = BatchEvaluationMapper.ToRuntimeRequests(request.TenantId, pageItems);
var results = new List<BatchEvaluationResultDto>(runtimeRequests.Count);
var cacheHits = 0;
var cacheMisses = 0;
var processed = 0;
foreach (var runtimeRequest in runtimeRequests)
{
if (budgetMs is int budget && sw.ElapsedMilliseconds >= budget)
{
break;
}
var response = await evaluator.EvaluateAsync(runtimeRequest, cancellationToken).ConfigureAwait(false);
processed++;
if (response.Cached)
{
cacheHits++;
}
else
{
cacheMisses++;
}
results.Add(new BatchEvaluationResultDto(
response.PackId,
response.Version,
response.PolicyDigest,
response.Status,
response.Severity,
response.RuleName,
response.Priority,
response.Annotations,
response.Warnings,
response.AppliedException,
response.CorrelationId,
response.Cached,
response.CacheSource,
response.EvaluationDurationMs));
}
var nextOffset = offset + processed;
string? nextPageToken = null;
if (nextOffset < request.Items.Count)
{
nextPageToken = nextOffset.ToString();
}
var budgetRemaining = budgetMs is int budgetValue
? Math.Max(0, budgetValue - sw.ElapsedMilliseconds)
: (long?)null;
var responsePayload = new BatchEvaluationResponseDto(
Results: results,
NextPageToken: nextPageToken,
Total: request.Items.Count,
Returned: processed,
CacheHits: cacheHits,
CacheMisses: cacheMisses,
DurationMs: sw.ElapsedMilliseconds,
BudgetRemainingMs: budgetRemaining);
return Results.Ok(responsePayload);
}
private static bool TryParseOffset(string? token, out int offset)
{
if (string.IsNullOrWhiteSpace(token))
{
offset = 0;
return true;
}
return int.TryParse(token, out offset) && offset >= 0;
}
}

View File

@@ -0,0 +1,42 @@
using Microsoft.Extensions.Logging;
using StellaOps.Policy.Engine.ExceptionCache;
namespace StellaOps.Policy.Engine.Events;
/// <summary>
/// Publishes exception lifecycle events and keeps local caches warm.
/// </summary>
public interface IExceptionEventPublisher
{
Task PublishAsync(ExceptionEvent exceptionEvent, CancellationToken cancellationToken = default);
}
internal sealed class LoggingExceptionEventPublisher : IExceptionEventPublisher
{
private readonly IExceptionEffectiveCache? _cache;
private readonly ILogger<LoggingExceptionEventPublisher> _logger;
public LoggingExceptionEventPublisher(
IExceptionEffectiveCache? cache,
ILogger<LoggingExceptionEventPublisher> logger)
{
_cache = cache;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task PublishAsync(ExceptionEvent exceptionEvent, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(exceptionEvent);
if (_cache is not null)
{
await _cache.HandleExceptionEventAsync(exceptionEvent, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation(
"Published exception event {EventType} for exception {ExceptionId} tenant {TenantId}",
exceptionEvent.EventType,
exceptionEvent.ExceptionId,
exceptionEvent.TenantId);
}
}

View File

@@ -0,0 +1,45 @@
namespace StellaOps.Policy.Engine.Options;
/// <summary>
/// Options controlling the exception activation/expiry lifecycle worker.
/// </summary>
public sealed class PolicyEngineExceptionLifecycleOptions
{
/// <summary>Polling interval for lifecycle checks.</summary>
public int PollIntervalSeconds { get; set; } = 60;
/// <summary>How far back to look when picking up overdue activations.</summary>
public int ActivationLookbackMinutes { get; set; } = 5;
/// <summary>How far back to look when expiring exceptions.</summary>
public int ExpiryLookbackMinutes { get; set; } = 5;
/// <summary>How far ahead to look for upcoming expirations.</summary>
public int ExpiryHorizonMinutes { get; set; } = 5;
/// <summary>Maximum exceptions processed per cycle.</summary>
public int MaxBatchSize { get; set; } = 500;
public void Validate()
{
if (PollIntervalSeconds <= 0)
{
throw new InvalidOperationException("Exception lifecycle poll interval must be greater than zero.");
}
if (ActivationLookbackMinutes < 0 || ExpiryLookbackMinutes < 0 || ExpiryHorizonMinutes < 0)
{
throw new InvalidOperationException("Exception lifecycle windows cannot be negative.");
}
if (MaxBatchSize <= 0)
{
throw new InvalidOperationException("Exception lifecycle batch size must be greater than zero.");
}
}
public TimeSpan PollInterval => TimeSpan.FromSeconds(PollIntervalSeconds);
public TimeSpan ActivationLookback => TimeSpan.FromMinutes(ActivationLookbackMinutes);
public TimeSpan ExpiryLookback => TimeSpan.FromMinutes(ExpiryLookbackMinutes);
public TimeSpan ExpiryHorizon => TimeSpan.FromMinutes(ExpiryHorizonMinutes);
}

View File

@@ -33,11 +33,13 @@ public sealed class PolicyEngineOptions
public ReachabilityFactsCacheOptions ReachabilityCache { get; } = new();
public PolicyEvaluationCacheOptions EvaluationCache { get; } = new();
public EffectiveDecisionMapOptions EffectiveDecisionMap { get; } = new();
public ExceptionCacheOptions ExceptionCache { get; } = new();
public PolicyEvaluationCacheOptions EvaluationCache { get; } = new();
public EffectiveDecisionMapOptions EffectiveDecisionMap { get; } = new();
public ExceptionCacheOptions ExceptionCache { get; } = new();
public PolicyEngineExceptionLifecycleOptions ExceptionLifecycle { get; } = new();
public void Validate()
{
@@ -45,12 +47,13 @@ public sealed class PolicyEngineOptions
Storage.Validate();
Workers.Validate();
ResourceServer.Validate();
Compilation.Validate();
Activation.Validate();
Telemetry.Validate();
RiskProfile.Validate();
}
}
Compilation.Validate();
Activation.Validate();
Telemetry.Validate();
RiskProfile.Validate();
ExceptionLifecycle.Validate();
}
}
public sealed class PolicyEngineAuthorityOptions
{

View File

@@ -5,18 +5,22 @@ using StellaOps.Auth.Abstractions;
using StellaOps.Auth.Client;
using StellaOps.Auth.ServerIntegration;
using StellaOps.Configuration;
using StellaOps.Policy.Engine.Hosting;
using StellaOps.Policy.Engine.Options;
using StellaOps.Policy.Engine.Compilation;
using StellaOps.Policy.Engine.Endpoints;
using StellaOps.PolicyDsl;
using StellaOps.Policy.Engine.Services;
using StellaOps.Policy.Engine.Workers;
using StellaOps.Policy.Engine.Streaming;
using StellaOps.Policy.Engine.Telemetry;
using StellaOps.AirGap.Policy;
using StellaOps.Policy.Engine.Orchestration;
using StellaOps.Policy.Engine.ReachabilityFacts;
using StellaOps.Policy.Engine.Hosting;
using StellaOps.Policy.Engine.Options;
using StellaOps.Policy.Engine.Compilation;
using StellaOps.Policy.Engine.Endpoints;
using StellaOps.Policy.Engine.BatchEvaluation;
using StellaOps.Policy.Engine.DependencyInjection;
using StellaOps.PolicyDsl;
using StellaOps.Policy.Engine.Services;
using StellaOps.Policy.Engine.Workers;
using StellaOps.Policy.Engine.Streaming;
using StellaOps.Policy.Engine.Telemetry;
using StellaOps.AirGap.Policy;
using StellaOps.Policy.Engine.Orchestration;
using StellaOps.Policy.Engine.ReachabilityFacts;
using StellaOps.Policy.Engine.Storage.InMemory;
using StellaOps.Policy.Engine.Storage.Mongo.Repositories;
var builder = WebApplication.CreateBuilder(args);
@@ -108,9 +112,10 @@ builder.Services.AddOptions<PolicyEngineOptions>()
})
.ValidateOnStart();
builder.Services.AddSingleton(sp => sp.GetRequiredService<IOptions<PolicyEngineOptions>>().Value);
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddSingleton<PolicyEngineStartupDiagnostics>();
builder.Services.AddSingleton(sp => sp.GetRequiredService<IOptions<PolicyEngineOptions>>().Value);
builder.Services.AddSingleton(sp => sp.GetRequiredService<PolicyEngineOptions>().ExceptionLifecycle);
builder.Services.AddSingleton(TimeProvider.System);
builder.Services.AddSingleton<PolicyEngineStartupDiagnostics>();
builder.Services.AddSingleton<PolicyTimelineEvents>();
builder.Services.AddSingleton<EvidenceBundleService>();
builder.Services.AddSingleton<PolicyEvaluationAttestationService>();
@@ -123,41 +128,50 @@ builder.Services.AddSingleton<StellaOps.Policy.Engine.Scoring.IRiskScoringJobSto
builder.Services.AddSingleton<StellaOps.Policy.Engine.Scoring.RiskScoringTriggerService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Simulation.RiskSimulationService>();
builder.Services.AddSingleton<StellaOps.Policy.RiskProfile.Export.ProfileExportService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Events.ProfileEventPublisher>();
builder.Services.AddHostedService<IncidentModeExpirationWorker>();
builder.Services.AddHostedService<PolicyEngineBootstrapWorker>();
builder.Services.AddSingleton<StellaOps.PolicyDsl.PolicyCompiler>();
builder.Services.AddSingleton<PolicyCompilationService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Services.PathScopeMetrics>();
builder.Services.AddSingleton<PolicyEvaluationService>();
builder.Services.AddSingleton<PathScopeSimulationService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Overlay.OverlayProjectionService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Overlay.IOverlayEventSink, StellaOps.Policy.Engine.Overlay.LoggingOverlayEventSink>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Overlay.OverlayChangeEventPublisher>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Overlay.PathScopeSimulationBridgeService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Events.ProfileEventPublisher>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Events.IExceptionEventPublisher>(sp =>
new StellaOps.Policy.Engine.Events.LoggingExceptionEventPublisher(
sp.GetService<StellaOps.Policy.Engine.ExceptionCache.IExceptionEffectiveCache>(),
sp.GetRequiredService<ILogger<StellaOps.Policy.Engine.Events.LoggingExceptionEventPublisher>>()));
builder.Services.AddSingleton<ExceptionLifecycleService>();
builder.Services.AddHostedService<ExceptionLifecycleWorker>();
builder.Services.AddHostedService<IncidentModeExpirationWorker>();
builder.Services.AddHostedService<PolicyEngineBootstrapWorker>();
builder.Services.AddSingleton<StellaOps.PolicyDsl.PolicyCompiler>();
builder.Services.AddSingleton<PolicyCompilationService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Services.PathScopeMetrics>();
builder.Services.AddSingleton<PolicyEvaluationService>();
builder.Services.AddPolicyEngineCore();
builder.Services.AddSingleton<PathScopeSimulationService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Overlay.OverlayProjectionService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Overlay.IOverlayEventSink, StellaOps.Policy.Engine.Overlay.LoggingOverlayEventSink>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Overlay.OverlayChangeEventPublisher>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Overlay.PathScopeSimulationBridgeService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.TrustWeighting.TrustWeightingService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.AdvisoryAI.AdvisoryAiKnobsService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.BatchContext.BatchContextService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Services.EvidenceSummaryService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Services.PolicyBundleService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Services.PolicyRuntimeEvaluator>();
builder.Services.AddSingleton<IPolicyPackRepository, InMemoryPolicyPackRepository>();
builder.Services.AddSingleton<IOrchestratorJobStore, InMemoryOrchestratorJobStore>();
builder.Services.AddSingleton<OrchestratorJobService>();
builder.Services.AddSingleton<IWorkerResultStore, InMemoryWorkerResultStore>();
builder.Services.AddSingleton<PolicyWorkerService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Ledger.ILedgerExportStore, StellaOps.Policy.Engine.Ledger.InMemoryLedgerExportStore>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Ledger.LedgerExportService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Snapshots.ISnapshotStore, StellaOps.Policy.Engine.Snapshots.InMemorySnapshotStore>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Snapshots.SnapshotService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Violations.IViolationEventStore, StellaOps.Policy.Engine.Violations.InMemoryViolationEventStore>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Violations.ViolationEventService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Violations.SeverityFusionService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Violations.ConflictHandlingService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Services.PolicyDecisionService>();
builder.Services.AddSingleton<IReachabilityFactsStore, InMemoryReachabilityFactsStore>();
builder.Services.AddSingleton<IReachabilityFactsOverlayCache, InMemoryReachabilityFactsOverlayCache>();
builder.Services.AddSingleton<ReachabilityFactsJoiningService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Services.PolicyRuntimeEvaluator>();
builder.Services.AddSingleton<IPolicyPackRepository, InMemoryPolicyPackRepository>();
builder.Services.AddSingleton<IOrchestratorJobStore, InMemoryOrchestratorJobStore>();
builder.Services.AddSingleton<OrchestratorJobService>();
builder.Services.AddSingleton<IWorkerResultStore, InMemoryWorkerResultStore>();
builder.Services.AddSingleton<PolicyWorkerService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Ledger.ILedgerExportStore, StellaOps.Policy.Engine.Ledger.InMemoryLedgerExportStore>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Ledger.LedgerExportService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Snapshots.ISnapshotStore, StellaOps.Policy.Engine.Snapshots.InMemorySnapshotStore>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Snapshots.SnapshotService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Violations.IViolationEventStore, StellaOps.Policy.Engine.Violations.InMemoryViolationEventStore>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Violations.ViolationEventService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Violations.SeverityFusionService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Violations.ConflictHandlingService>();
builder.Services.AddSingleton<StellaOps.Policy.Engine.Services.PolicyDecisionService>();
builder.Services.AddSingleton<IExceptionRepository, InMemoryExceptionRepository>();
builder.Services.AddSingleton<IReachabilityFactsStore, InMemoryReachabilityFactsStore>();
builder.Services.AddSingleton<IReachabilityFactsOverlayCache, InMemoryReachabilityFactsOverlayCache>();
builder.Services.AddSingleton<ReachabilityFactsJoiningService>();
builder.Services.AddSingleton<IRuntimeEvaluationExecutor, RuntimeEvaluationExecutor>();
builder.Services.AddHttpContextAccessor();
builder.Services.AddRouting(options => options.LowercaseUrls = true);
@@ -202,13 +216,14 @@ app.MapGet("/readyz", (PolicyEngineStartupDiagnostics diagnostics) =>
app.MapGet("/", () => Results.Redirect("/healthz"));
app.MapPolicyCompilation();
app.MapPolicyPacks();
app.MapPathScopeSimulation();
app.MapOverlaySimulation();
app.MapEvidenceSummaries();
app.MapTrustWeighting();
app.MapAdvisoryAiKnobs();
app.MapPolicyCompilation();
app.MapPolicyPacks();
app.MapPathScopeSimulation();
app.MapOverlaySimulation();
app.MapEvidenceSummaries();
app.MapBatchEvaluation();
app.MapTrustWeighting();
app.MapAdvisoryAiKnobs();
app.MapBatchContext();
app.MapOrchestratorJobs();
app.MapPolicyWorker();

View File

@@ -4,11 +4,11 @@ This service hosts the Policy Engine APIs and background workers introduced in *
## Compliance Checklist
- [x] Configuration loads from `policy-engine.yaml`/environment variables and validates on startup.
- [x] Authority client scaffolding enforces `policy:*` + `effective:write` scopes and respects back-channel timeouts.
- [x] Resource server authentication requires Policy Engine scopes with tenant-aware policies.
- [x] Health and readiness endpoints exist for platform probes.
- [ ] Deterministic policy evaluation pipeline implemented (POLICY-ENGINE-20-002).
- [ ] Mongo materialisation writers implemented (POLICY-ENGINE-20-004).
- [ ] Observability (metrics/traces/logs) completed (POLICY-ENGINE-20-007).
- [ ] Comprehensive test suites and perf baselines established (POLICY-ENGINE-20-008).
- [x] Configuration loads from `policy-engine.yaml`/environment variables and validates on startup.
- [x] Authority client scaffolding enforces `policy:*` + `effective:write` scopes and respects back-channel timeouts.
- [x] Resource server authentication requires Policy Engine scopes with tenant-aware policies.
- [x] Health and readiness endpoints exist for platform probes.
- [x] Deterministic policy evaluation pipeline implemented (POLICY-ENGINE-20-002).
- [x] Mongo materialisation writers implemented (POLICY-ENGINE-20-004).
- [x] Observability (metrics/traces/logs) completed (POLICY-ENGINE-20-007).
- [x] Comprehensive test suites and perf baselines established (POLICY-ENGINE-20-008).

View File

@@ -1,3 +1,4 @@
using System.Buffers.Binary;
using System.Security.Cryptography;
using System.Text;
using StellaOps.Policy.Engine.Domain;
@@ -62,7 +63,7 @@ internal sealed class EvidenceSummaryService
private DateTimeOffset DeriveIngestedAt(byte[] hashBytes)
{
// Use a deterministic timestamp within the last 30 days to avoid non-determinism in tests.
var seconds = BitConverter.ToUInt32(hashBytes, 0) % (30u * 24u * 60u * 60u);
var seconds = BinaryPrimitives.ReadUInt32BigEndian(hashBytes) % (30u * 24u * 60u * 60u);
var baseline = _timeProvider.GetUtcNow().UtcDateTime.Date; // midnight UTC today
var dt = baseline.AddSeconds(seconds);
return new DateTimeOffset(dt, TimeSpan.Zero);

View File

@@ -58,6 +58,7 @@ internal sealed class PolicyRuntimeEvaluationService
private readonly IPolicyPackRepository _repository;
private readonly IPolicyEvaluationCache _cache;
private readonly PolicyEvaluator _evaluator;
private readonly ReachabilityFacts.ReachabilityFactsJoiningService? _reachabilityFacts;
private readonly TimeProvider _timeProvider;
private readonly ILogger<PolicyRuntimeEvaluationService> _logger;
@@ -71,12 +72,14 @@ internal sealed class PolicyRuntimeEvaluationService
IPolicyPackRepository repository,
IPolicyEvaluationCache cache,
PolicyEvaluator evaluator,
ReachabilityFacts.ReachabilityFactsJoiningService? reachabilityFacts,
TimeProvider timeProvider,
ILogger<PolicyRuntimeEvaluationService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_evaluator = evaluator ?? throw new ArgumentNullException(nameof(evaluator));
_reachabilityFacts = reachabilityFacts;
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -90,35 +93,38 @@ internal sealed class PolicyRuntimeEvaluationService
{
ArgumentNullException.ThrowIfNull(request);
using var activity = PolicyEngineTelemetry.StartEvaluateActivity(
request.TenantId, request.PackId, runId: null);
activity?.SetTag("policy.version", request.Version);
activity?.SetTag("subject.purl", request.SubjectPurl);
activity?.SetTag("advisory.id", request.AdvisoryId);
var startTimestamp = _timeProvider.GetTimestamp();
var evaluationTimestamp = request.EvaluationTimestamp ?? _timeProvider.GetUtcNow();
var effectiveRequest = _reachabilityFacts is null
? request
: await EnrichReachabilityAsync(request, cancellationToken).ConfigureAwait(false);
using var activity = PolicyEngineTelemetry.StartEvaluateActivity(
effectiveRequest.TenantId, effectiveRequest.PackId, runId: null);
activity?.SetTag("policy.version", effectiveRequest.Version);
activity?.SetTag("subject.purl", effectiveRequest.SubjectPurl);
activity?.SetTag("advisory.id", effectiveRequest.AdvisoryId);
// Load the compiled policy bundle
var bundle = await _repository.GetBundleAsync(request.PackId, request.Version, cancellationToken)
var bundle = await _repository.GetBundleAsync(effectiveRequest.PackId, effectiveRequest.Version, cancellationToken)
.ConfigureAwait(false);
if (bundle is null)
{
PolicyEngineTelemetry.RecordError("evaluation", request.TenantId);
PolicyEngineTelemetry.RecordEvaluationFailure(request.TenantId, request.PackId, "bundle_not_found");
PolicyEngineTelemetry.RecordError("evaluation", effectiveRequest.TenantId);
PolicyEngineTelemetry.RecordEvaluationFailure(effectiveRequest.TenantId, effectiveRequest.PackId, "bundle_not_found");
activity?.SetStatus(ActivityStatusCode.Error, "Bundle not found");
throw new InvalidOperationException(
$"Policy bundle not found for pack '{request.PackId}' version {request.Version}.");
$"Policy bundle not found for pack '{effectiveRequest.PackId}' version {effectiveRequest.Version}.");
}
// Compute deterministic cache key
var subjectDigest = ComputeSubjectDigest(request.TenantId, request.SubjectPurl, request.AdvisoryId);
var contextDigest = ComputeContextDigest(request);
var subjectDigest = ComputeSubjectDigest(effectiveRequest.TenantId, effectiveRequest.SubjectPurl, effectiveRequest.AdvisoryId);
var contextDigest = ComputeContextDigest(effectiveRequest);
var cacheKey = PolicyEvaluationCacheKey.Create(bundle.Digest, subjectDigest, contextDigest);
// Try cache lookup unless bypassed
if (!request.BypassCache)
if (!effectiveRequest.BypassCache)
{
var cacheResult = await _cache.GetAsync(cacheKey, cancellationToken).ConfigureAwait(false);
if (cacheResult.CacheHit && cacheResult.Entry is not null)
@@ -132,10 +138,10 @@ internal sealed class PolicyRuntimeEvaluationService
activity?.SetStatus(ActivityStatusCode.Ok);
_logger.LogDebug(
"Cache hit for evaluation {PackId}@{Version} subject {Subject} from {Source}",
request.PackId, request.Version, request.SubjectPurl, cacheResult.Source);
effectiveRequest.PackId, effectiveRequest.Version, effectiveRequest.SubjectPurl, cacheResult.Source);
return CreateResponseFromCache(
request, bundle.Digest, cacheResult.Entry, cacheResult.Source, duration);
effectiveRequest, bundle.Digest, cacheResult.Entry, cacheResult.Source, duration);
}
}
@@ -153,13 +159,13 @@ internal sealed class PolicyRuntimeEvaluationService
}
var context = new PolicyEvaluationContext(
request.Severity,
effectiveRequest.Severity,
new PolicyEvaluationEnvironment(ImmutableDictionary<string, string>.Empty),
request.Advisory,
request.Vex,
request.Sbom,
request.Exceptions,
request.Reachability,
effectiveRequest.Advisory,
effectiveRequest.Vex,
effectiveRequest.Sbom,
effectiveRequest.Exceptions,
effectiveRequest.Reachability,
evaluationTimestamp);
var evalRequest = new Evaluation.PolicyEvaluationRequest(document, context);
@@ -187,11 +193,25 @@ internal sealed class PolicyRuntimeEvaluationService
var evalDurationSeconds = evalDuration / 1000.0;
// Record metrics
PolicyEngineTelemetry.RecordEvaluationLatency(evalDurationSeconds, request.TenantId, request.PackId);
PolicyEngineTelemetry.RecordEvaluation(request.TenantId, request.PackId, "full");
PolicyEngineTelemetry.RecordEvaluationLatency(evalDurationSeconds, effectiveRequest.TenantId, effectiveRequest.PackId);
PolicyEngineTelemetry.RecordEvaluation(effectiveRequest.TenantId, effectiveRequest.PackId, "full");
if (!string.IsNullOrEmpty(result.RuleName))
{
PolicyEngineTelemetry.RecordRuleFired(request.PackId, result.RuleName);
PolicyEngineTelemetry.RecordRuleFired(effectiveRequest.PackId, result.RuleName);
}
if (result.AppliedException is not null)
{
PolicyEngineTelemetry.RecordExceptionApplication(effectiveRequest.TenantId, result.AppliedException.EffectType.ToString());
PolicyEngineTelemetry.RecordExceptionApplicationLatency(evalDurationSeconds, effectiveRequest.TenantId, result.AppliedException.EffectType.ToString());
_logger.LogInformation(
"Applied exception {ExceptionId} (effect {EffectType}) for tenant {TenantId} pack {PackId}@{Version} aoc {CompilationId}",
result.AppliedException.ExceptionId,
result.AppliedException.EffectType,
effectiveRequest.TenantId,
effectiveRequest.PackId,
effectiveRequest.Version,
bundle.AocMetadata?.CompilationId ?? "none");
}
activity?.SetTag("evaluation.status", result.Status);
@@ -201,11 +221,11 @@ internal sealed class PolicyRuntimeEvaluationService
_logger.LogDebug(
"Evaluated {PackId}@{Version} subject {Subject} in {Duration}ms - {Status}",
request.PackId, request.Version, request.SubjectPurl, evalDuration, result.Status);
effectiveRequest.PackId, effectiveRequest.Version, effectiveRequest.SubjectPurl, evalDuration, result.Status);
return new RuntimeEvaluationResponse(
request.PackId,
request.Version,
effectiveRequest.PackId,
effectiveRequest.Version,
bundle.Digest,
result.Status,
result.Severity,
@@ -240,8 +260,12 @@ internal sealed class PolicyRuntimeEvaluationService
var cacheHits = 0;
var cacheMisses = 0;
var hydratedRequests = _reachabilityFacts is null
? requests
: await EnrichReachabilityBatchAsync(requests, cancellationToken).ConfigureAwait(false);
// Group by pack/version for bundle loading efficiency
var groups = requests.GroupBy(r => (r.PackId, r.Version));
var groups = hydratedRequests.GroupBy(r => (r.PackId, r.Version));
foreach (var group in groups)
{
@@ -351,6 +375,20 @@ internal sealed class PolicyRuntimeEvaluationService
PolicyEngineTelemetry.RecordRuleFired(packId, result.RuleName);
}
if (result.AppliedException is not null)
{
PolicyEngineTelemetry.RecordExceptionApplication(request.TenantId, result.AppliedException.EffectType.ToString());
PolicyEngineTelemetry.RecordExceptionApplicationLatency(duration / 1000.0, request.TenantId, result.AppliedException.EffectType.ToString());
_logger.LogInformation(
"Applied exception {ExceptionId} (effect {EffectType}) for tenant {TenantId} pack {PackId}@{Version} aoc {CompilationId}",
result.AppliedException.ExceptionId,
result.AppliedException.EffectType,
request.TenantId,
request.PackId,
request.Version,
bundle.AocMetadata?.CompilationId ?? "none");
}
results.Add(new RuntimeEvaluationResponse(
request.PackId,
request.Version,
@@ -448,7 +486,15 @@ internal sealed class PolicyRuntimeEvaluationService
vexStatements = request.Vex.Statements.Select(s => $"{s.Status}:{s.Justification}").OrderBy(s => s).ToArray(),
sbomTags = request.Sbom.Tags.OrderBy(t => t).ToArray(),
exceptionCount = request.Exceptions.Instances.Length,
reachability = request.Reachability.State,
reachability = new
{
state = request.Reachability.State,
confidence = request.Reachability.Confidence,
score = request.Reachability.Score,
hasRuntimeEvidence = request.Reachability.HasRuntimeEvidence,
source = request.Reachability.Source,
method = request.Reachability.Method
},
};
var json = JsonSerializer.Serialize(contextData, ContextSerializerOptions);
@@ -470,5 +516,98 @@ internal sealed class PolicyRuntimeEvaluationService
var elapsed = _timeProvider.GetElapsedTime(startTimestamp);
return (long)elapsed.TotalMilliseconds;
}
private async Task<RuntimeEvaluationRequest> EnrichReachabilityAsync(
RuntimeEvaluationRequest request,
CancellationToken cancellationToken)
{
if (_reachabilityFacts is null || !request.Reachability.IsUnknown)
{
return request;
}
var fact = await _reachabilityFacts
.GetFactAsync(request.TenantId, request.SubjectPurl, request.AdvisoryId, cancellationToken)
.ConfigureAwait(false);
if (fact is null)
{
return request;
}
var reachability = new PolicyEvaluationReachability(
State: fact.State.ToString().ToLowerInvariant(),
Confidence: fact.Confidence,
Score: fact.Score,
HasRuntimeEvidence: fact.HasRuntimeEvidence,
Source: fact.Source,
Method: fact.Method.ToString().ToLowerInvariant(),
EvidenceRef: fact.EvidenceRef ?? fact.EvidenceHash);
ReachabilityFacts.ReachabilityFactsTelemetry.RecordFactApplied(reachability.State);
return request with { Reachability = reachability };
}
private async Task<IReadOnlyList<RuntimeEvaluationRequest>> EnrichReachabilityBatchAsync(
IReadOnlyList<RuntimeEvaluationRequest> requests,
CancellationToken cancellationToken)
{
if (_reachabilityFacts is null)
{
return requests;
}
var enriched = new List<RuntimeEvaluationRequest>(requests.Count);
foreach (var tenantGroup in requests.GroupBy(r => r.TenantId, StringComparer.Ordinal))
{
var pending = tenantGroup
.Where(r => r.Reachability.IsUnknown)
.Select(r => new ReachabilityFacts.ReachabilityFactsRequest(r.SubjectPurl, r.AdvisoryId))
.Distinct()
.ToList();
ReachabilityFacts.ReachabilityFactsBatch? batch = null;
if (pending.Count > 0)
{
batch = await _reachabilityFacts
.GetFactsBatchAsync(tenantGroup.Key, pending, cancellationToken)
.ConfigureAwait(false);
}
var lookup = batch?.Found ?? new Dictionary<ReachabilityFacts.ReachabilityFactKey, ReachabilityFacts.ReachabilityFact>();
foreach (var request in tenantGroup)
{
if (!request.Reachability.IsUnknown)
{
enriched.Add(request);
continue;
}
var key = new ReachabilityFacts.ReachabilityFactKey(request.TenantId, request.SubjectPurl, request.AdvisoryId);
if (lookup.TryGetValue(key, out var fact))
{
var reachability = new PolicyEvaluationReachability(
State: fact.State.ToString().ToLowerInvariant(),
Confidence: fact.Confidence,
Score: fact.Score,
HasRuntimeEvidence: fact.HasRuntimeEvidence,
Source: fact.Source,
Method: fact.Method.ToString().ToLowerInvariant(),
EvidenceRef: fact.EvidenceRef ?? fact.EvidenceHash);
ReachabilityFacts.ReachabilityFactsTelemetry.RecordFactApplied(reachability.State);
enriched.Add(request with { Reachability = reachability });
}
else
{
enriched.Add(request);
}
}
}
return enriched;
}
}

View File

@@ -0,0 +1,351 @@
using System.Collections.Immutable;
using System.Collections.Concurrent;
using System.Linq;
using StellaOps.Policy.Engine.Storage.Mongo.Documents;
using StellaOps.Policy.Engine.Storage.Mongo.Repositories;
namespace StellaOps.Policy.Engine.Storage.InMemory;
/// <summary>
/// In-memory implementation of IExceptionRepository for offline/test runs.
/// Provides minimal semantics needed for lifecycle processing.
/// </summary>
public sealed class InMemoryExceptionRepository : IExceptionRepository
{
private readonly ConcurrentDictionary<(string Tenant, string Id), PolicyExceptionDocument> _exceptions = new();
private readonly ConcurrentDictionary<(string Tenant, string Id), ExceptionBindingDocument> _bindings = new();
public Task<PolicyExceptionDocument> CreateExceptionAsync(PolicyExceptionDocument exception, CancellationToken cancellationToken)
{
_exceptions[(exception.TenantId.ToLowerInvariant(), exception.Id)] = Clone(exception);
return Task.FromResult(exception);
}
public Task<PolicyExceptionDocument?> GetExceptionAsync(string tenantId, string exceptionId, CancellationToken cancellationToken)
{
_exceptions.TryGetValue((tenantId.ToLowerInvariant(), exceptionId), out var value);
return Task.FromResult(value is null ? null : Clone(value));
}
public Task<PolicyExceptionDocument?> UpdateExceptionAsync(PolicyExceptionDocument exception, CancellationToken cancellationToken)
{
_exceptions[(exception.TenantId.ToLowerInvariant(), exception.Id)] = Clone(exception);
return Task.FromResult<PolicyExceptionDocument?>(exception);
}
public Task<ImmutableArray<PolicyExceptionDocument>> ListExceptionsAsync(ExceptionQueryOptions options, CancellationToken cancellationToken)
{
var query = _exceptions.Values.AsEnumerable();
if (options.Statuses.Any())
{
query = query.Where(e => options.Statuses.Contains(e.Status, StringComparer.OrdinalIgnoreCase));
}
if (options.Types.Any())
{
query = query.Where(e => options.Types.Contains(e.ExceptionType, StringComparer.OrdinalIgnoreCase));
}
return Task.FromResult(query.Select(Clone).ToImmutableArray());
}
public Task<ImmutableArray<PolicyExceptionDocument>> ListExceptionsAsync(string tenantId, ExceptionQueryOptions options, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var scoped = _exceptions.Values.Where(e => e.TenantId.Equals(tenant, StringComparison.OrdinalIgnoreCase)).ToList();
var result = scoped.AsEnumerable();
if (options.Statuses.Any())
{
result = result.Where(e => options.Statuses.Contains(e.Status, StringComparer.OrdinalIgnoreCase));
}
if (options.Types.Any())
{
result = result.Where(e => options.Types.Contains(e.ExceptionType, StringComparer.OrdinalIgnoreCase));
}
return Task.FromResult(result.Select(Clone).ToImmutableArray());
}
public Task<ImmutableArray<PolicyExceptionDocument>> FindApplicableExceptionsAsync(string tenantId, ExceptionQueryOptions options, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var results = _exceptions.Values
.Where(e => e.TenantId.Equals(tenant, StringComparison.OrdinalIgnoreCase))
.Where(e => e.Status.Equals("active", StringComparison.OrdinalIgnoreCase))
.Select(Clone)
.ToImmutableArray();
return Task.FromResult(results);
}
public Task<bool> UpdateExceptionStatusAsync(string tenantId, string exceptionId, string newStatus, DateTimeOffset timestamp, CancellationToken cancellationToken)
{
var key = (tenantId.ToLowerInvariant(), exceptionId);
if (!_exceptions.TryGetValue(key, out var existing))
{
return Task.FromResult(false);
}
var updated = Clone(existing);
updated.Status = newStatus;
updated.UpdatedAt = timestamp;
if (newStatus == "active")
{
updated.ActivatedAt = timestamp;
}
if (newStatus == "expired")
{
updated.RevokedAt = timestamp;
}
_exceptions[key] = updated;
return Task.FromResult(true);
}
public Task<bool> RevokeExceptionAsync(string tenantId, string exceptionId, string revokedBy, string? reason, DateTimeOffset timestamp, CancellationToken cancellationToken)
{
return UpdateExceptionStatusAsync(tenantId, exceptionId, "revoked", timestamp, cancellationToken);
}
public Task<ImmutableArray<PolicyExceptionDocument>> GetExpiringExceptionsAsync(string tenantId, DateTimeOffset from, DateTimeOffset to, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var results = _exceptions.Values
.Where(e => e.TenantId.Equals(tenant, StringComparison.OrdinalIgnoreCase))
.Where(e => e.Status.Equals("active", StringComparison.OrdinalIgnoreCase))
.Where(e => e.ExpiresAt is not null && e.ExpiresAt >= from && e.ExpiresAt <= to)
.Select(Clone)
.ToImmutableArray();
return Task.FromResult(results);
}
public Task<ImmutableArray<PolicyExceptionDocument>> GetPendingActivationsAsync(string tenantId, DateTimeOffset asOf, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var results = _exceptions.Values
.Where(e => e.TenantId.Equals(tenant, StringComparison.OrdinalIgnoreCase))
.Where(e => e.Status.Equals("approved", StringComparison.OrdinalIgnoreCase))
.Where(e => e.EffectiveFrom is null || e.EffectiveFrom <= asOf)
.Select(Clone)
.ToImmutableArray();
return Task.FromResult(results);
}
public Task<ExceptionReviewDocument> CreateReviewAsync(ExceptionReviewDocument review, CancellationToken cancellationToken)
{
return Task.FromResult(review);
}
public Task<ExceptionReviewDocument?> GetReviewAsync(string tenantId, string reviewId, CancellationToken cancellationToken)
{
return Task.FromResult<ExceptionReviewDocument?>(null);
}
public Task<ExceptionReviewDocument?> AddReviewDecisionAsync(string tenantId, string reviewId, ReviewDecisionDocument decision, CancellationToken cancellationToken)
{
return Task.FromResult<ExceptionReviewDocument?>(null);
}
public Task<ExceptionReviewDocument?> CompleteReviewAsync(string tenantId, string reviewId, string finalStatus, DateTimeOffset completedAt, CancellationToken cancellationToken)
{
return Task.FromResult<ExceptionReviewDocument?>(null);
}
public Task<ImmutableArray<ExceptionReviewDocument>> GetReviewsForExceptionAsync(string tenantId, string exceptionId, CancellationToken cancellationToken)
{
return Task.FromResult(ImmutableArray<ExceptionReviewDocument>.Empty);
}
public Task<ImmutableArray<ExceptionReviewDocument>> GetPendingReviewsAsync(string tenantId, string? reviewerId, CancellationToken cancellationToken)
{
return Task.FromResult(ImmutableArray<ExceptionReviewDocument>.Empty);
}
public Task<ExceptionBindingDocument> UpsertBindingAsync(ExceptionBindingDocument binding, CancellationToken cancellationToken)
{
_bindings[(binding.TenantId.ToLowerInvariant(), binding.Id)] = Clone(binding);
return Task.FromResult(binding);
}
public Task<ImmutableArray<ExceptionBindingDocument>> GetBindingsForExceptionAsync(string tenantId, string exceptionId, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var results = _bindings.Values
.Where(b => b.TenantId.Equals(tenant, StringComparison.OrdinalIgnoreCase) && b.ExceptionId == exceptionId)
.Select(Clone)
.ToImmutableArray();
return Task.FromResult(results);
}
public Task<ImmutableArray<ExceptionBindingDocument>> GetActiveBindingsForAssetAsync(string tenantId, string assetId, DateTimeOffset asOf, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var results = _bindings.Values
.Where(b => b.TenantId.Equals(tenant, StringComparison.OrdinalIgnoreCase))
.Where(b => b.AssetId == assetId)
.Where(b => b.Status == "active")
.Where(b => b.EffectiveFrom <= asOf && (b.ExpiresAt is null || b.ExpiresAt > asOf))
.Select(Clone)
.ToImmutableArray();
return Task.FromResult(results);
}
public Task<long> DeleteBindingsForExceptionAsync(string tenantId, string exceptionId, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var removed = _bindings.Where(kvp => kvp.Key.Tenant == tenant && kvp.Value.ExceptionId == exceptionId).ToList();
foreach (var kvp in removed)
{
_bindings.TryRemove(kvp.Key, out _);
}
return Task.FromResult((long)removed.Count);
}
public Task<ImmutableArray<ExceptionBindingDocument>> GetExpiredBindingsAsync(string tenantId, DateTimeOffset asOf, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var results = _bindings.Values
.Where(b => b.TenantId.Equals(tenant, StringComparison.OrdinalIgnoreCase))
.Where(b => b.Status == "active")
.Where(b => b.ExpiresAt is not null && b.ExpiresAt < asOf)
.Select(Clone)
.ToImmutableArray();
return Task.FromResult(results);
}
public Task<IReadOnlyDictionary<string, int>> GetExceptionCountsByStatusAsync(string tenantId, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var counts = _exceptions.Values
.Where(e => e.TenantId.Equals(tenant, StringComparison.OrdinalIgnoreCase))
.GroupBy(e => e.Status)
.ToDictionary(g => g.Key, g => g.Count(), StringComparer.OrdinalIgnoreCase);
return Task.FromResult((IReadOnlyDictionary<string, int>)counts);
}
public Task<ImmutableArray<ExceptionBindingDocument>> GetExpiredBindingsAsync(string tenantId, DateTimeOffset asOf, int limit, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var results = _bindings.Values
.Where(b => string.Equals(b.TenantId, tenant, StringComparison.OrdinalIgnoreCase))
.Where(b => b.Status == "active")
.Where(b => b.ExpiresAt is not null && b.ExpiresAt < asOf)
.Take(limit)
.Select(Clone)
.ToImmutableArray();
return Task.FromResult(results);
}
public Task<bool> UpdateBindingStatusAsync(string tenantId, string bindingId, string newStatus, CancellationToken cancellationToken)
{
var key = _bindings.Keys.FirstOrDefault(k => string.Equals(k.Tenant, tenantId, StringComparison.OrdinalIgnoreCase) && k.Id == bindingId);
if (key == default)
{
return Task.FromResult(false);
}
if (_bindings.TryGetValue(key, out var binding))
{
var updated = Clone(binding);
updated.Status = newStatus;
_bindings[key] = updated;
return Task.FromResult(true);
}
return Task.FromResult(false);
}
public Task<ImmutableArray<PolicyExceptionDocument>> FindApplicableExceptionsAsync(string tenantId, string assetId, string? advisoryId, DateTimeOffset asOf, CancellationToken cancellationToken)
{
var tenant = tenantId.ToLowerInvariant();
var activeExceptions = _exceptions.Values
.Where(e => string.Equals(e.TenantId, tenant, StringComparison.OrdinalIgnoreCase))
.Where(e => e.Status.Equals("active", StringComparison.OrdinalIgnoreCase))
.Where(e => (e.EffectiveFrom is null || e.EffectiveFrom <= asOf) && (e.ExpiresAt is null || e.ExpiresAt > asOf))
.ToDictionary(e => e.Id, Clone);
if (activeExceptions.Count == 0)
{
return Task.FromResult(ImmutableArray<PolicyExceptionDocument>.Empty);
}
var matchingIds = _bindings.Values
.Where(b => string.Equals(b.TenantId, tenant, StringComparison.OrdinalIgnoreCase))
.Where(b => b.Status == "active")
.Where(b => b.EffectiveFrom <= asOf && (b.ExpiresAt is null || b.ExpiresAt > asOf))
.Where(b => b.AssetId == assetId)
.Where(b => advisoryId is null || string.IsNullOrEmpty(b.AdvisoryId) || b.AdvisoryId == advisoryId)
.Select(b => b.ExceptionId)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var ex in activeExceptions.Values)
{
if (ex.Scope.ApplyToAll)
{
matchingIds.Add(ex.Id);
}
else if (ex.Scope.AssetIds.Contains(assetId, StringComparer.OrdinalIgnoreCase))
{
matchingIds.Add(ex.Id);
}
else if (advisoryId is not null && ex.Scope.AdvisoryIds.Contains(advisoryId, StringComparer.OrdinalIgnoreCase))
{
matchingIds.Add(ex.Id);
}
}
var result = matchingIds
.Where(activeExceptions.ContainsKey)
.Select(id => activeExceptions[id])
.ToImmutableArray();
return Task.FromResult(result);
}
private static PolicyExceptionDocument Clone(PolicyExceptionDocument source)
{
return new PolicyExceptionDocument
{
Id = source.Id,
TenantId = source.TenantId,
Name = source.Name,
ExceptionType = source.ExceptionType,
Status = source.Status,
EffectiveFrom = source.EffectiveFrom,
ExpiresAt = source.ExpiresAt,
CreatedAt = source.CreatedAt,
UpdatedAt = source.UpdatedAt,
ActivatedAt = source.ActivatedAt,
RevokedAt = source.RevokedAt,
RevokedBy = source.RevokedBy,
RevocationReason = source.RevocationReason,
Scope = source.Scope,
RiskAssessment = source.RiskAssessment,
Tags = source.Tags,
};
}
private static ExceptionBindingDocument Clone(ExceptionBindingDocument source)
{
return new ExceptionBindingDocument
{
Id = source.Id,
TenantId = source.TenantId,
ExceptionId = source.ExceptionId,
AssetId = source.AssetId,
AdvisoryId = source.AdvisoryId,
Status = source.Status,
EffectiveFrom = source.EffectiveFrom,
ExpiresAt = source.ExpiresAt,
};
}
}

View File

@@ -33,13 +33,20 @@ internal interface IExceptionRepository
CancellationToken cancellationToken);
/// <summary>
/// Lists exceptions with filtering and pagination.
/// Lists exceptions for a tenant with filtering and pagination.
/// </summary>
Task<ImmutableArray<PolicyExceptionDocument>> ListExceptionsAsync(
string tenantId,
ExceptionQueryOptions options,
CancellationToken cancellationToken);
/// <summary>
/// Lists exceptions across all tenants with filtering and pagination.
/// </summary>
Task<ImmutableArray<PolicyExceptionDocument>> ListExceptionsAsync(
ExceptionQueryOptions options,
CancellationToken cancellationToken);
/// <summary>
/// Finds active exceptions that apply to a specific asset/advisory.
/// </summary>

View File

@@ -100,12 +100,50 @@ internal sealed class MongoExceptionRepository : IExceptionRepository
string tenantId,
ExceptionQueryOptions options,
CancellationToken cancellationToken)
{
var filter = BuildFilter(options, tenantId.ToLowerInvariant());
var sort = BuildSort(options);
var results = await Exceptions
.Find(filter)
.Sort(sort)
.Skip(options.Skip)
.Limit(options.Limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return results.ToImmutableArray();
}
public async Task<ImmutableArray<PolicyExceptionDocument>> ListExceptionsAsync(
ExceptionQueryOptions options,
CancellationToken cancellationToken)
{
var filter = BuildFilter(options, tenantId: null);
var sort = BuildSort(options);
var results = await Exceptions
.Find(filter)
.Sort(sort)
.Skip(options.Skip)
.Limit(options.Limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return results.ToImmutableArray();
}
private static FilterDefinition<PolicyExceptionDocument> BuildFilter(
ExceptionQueryOptions options,
string? tenantId)
{
var filterBuilder = Builders<PolicyExceptionDocument>.Filter;
var filters = new List<FilterDefinition<PolicyExceptionDocument>>
var filters = new List<FilterDefinition<PolicyExceptionDocument>>();
if (!string.IsNullOrWhiteSpace(tenantId))
{
filterBuilder.Eq(e => e.TenantId, tenantId.ToLowerInvariant())
};
filters.Add(filterBuilder.Eq(e => e.TenantId, tenantId));
}
if (options.Statuses.Length > 0)
{
@@ -135,21 +173,19 @@ internal sealed class MongoExceptionRepository : IExceptionRepository
filterBuilder.Gt(e => e.ExpiresAt, now)));
}
var filter = filterBuilder.And(filters);
if (filters.Count == 0)
{
return FilterDefinition<PolicyExceptionDocument>.Empty;
}
var sort = options.SortDirection.Equals("asc", StringComparison.OrdinalIgnoreCase)
return filterBuilder.And(filters);
}
private static SortDefinition<PolicyExceptionDocument> BuildSort(ExceptionQueryOptions options)
{
return options.SortDirection.Equals("asc", StringComparison.OrdinalIgnoreCase)
? Builders<PolicyExceptionDocument>.Sort.Ascending(options.SortBy)
: Builders<PolicyExceptionDocument>.Sort.Descending(options.SortBy);
var results = await Exceptions
.Find(filter)
.Sort(sort)
.Skip(options.Skip)
.Limit(options.Limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return results.ToImmutableArray();
}
public async Task<ImmutableArray<PolicyExceptionDocument>> FindApplicableExceptionsAsync(

View File

@@ -363,12 +363,33 @@ public static class PolicyEngineTelemetry
/// </summary>
public static Counter<long> ExceptionOperations => ExceptionOperationsCounter;
// Counter: policy_exception_cache_operations_total{tenant,operation}
private static readonly Counter<long> ExceptionCacheOperationsCounter =
Meter.CreateCounter<long>(
"policy_exception_cache_operations_total",
unit: "operations",
description: "Total exception cache operations (hit, miss, set, warm, invalidate).");
// Counter: policy_exception_cache_operations_total{tenant,operation}
private static readonly Counter<long> ExceptionCacheOperationsCounter =
Meter.CreateCounter<long>(
"policy_exception_cache_operations_total",
unit: "operations",
description: "Total exception cache operations (hit, miss, set, warm, invalidate).");
// Counter: policy_exception_applications_total{tenant,effect}
private static readonly Counter<long> ExceptionApplicationsCounter =
Meter.CreateCounter<long>(
"policy_exception_applications_total",
unit: "applications",
description: "Total applied exceptions during evaluation by effect type.");
// Histogram: policy_exception_application_latency_seconds{tenant,effect}
private static readonly Histogram<double> ExceptionApplicationLatencyHistogram =
Meter.CreateHistogram<double>(
"policy_exception_application_latency_seconds",
unit: "s",
description: "Latency impact of exception application during evaluation.");
// Counter: policy_exception_lifecycle_total{tenant,event}
private static readonly Counter<long> ExceptionLifecycleCounter =
Meter.CreateCounter<long>(
"policy_exception_lifecycle_total",
unit: "events",
description: "Lifecycle events for exceptions (activated, expired, revoked).");
/// <summary>
/// Counter for exception cache operations.
@@ -611,16 +632,58 @@ public static class PolicyEngineTelemetry
/// </summary>
/// <param name="tenant">Tenant identifier.</param>
/// <param name="operation">Operation type (hit, miss, set, warm, invalidate_*, event_*).</param>
public static void RecordExceptionCacheOperation(string tenant, string operation)
{
var tags = new TagList
{
{ "tenant", NormalizeTenant(tenant) },
{ "operation", NormalizeTag(operation) },
};
ExceptionCacheOperationsCounter.Add(1, tags);
}
public static void RecordExceptionCacheOperation(string tenant, string operation)
{
var tags = new TagList
{
{ "tenant", NormalizeTenant(tenant) },
{ "operation", NormalizeTag(operation) },
};
ExceptionCacheOperationsCounter.Add(1, tags);
}
/// <summary>
/// Records that an exception was applied during evaluation.
/// </summary>
public static void RecordExceptionApplication(string tenant, string effectType)
{
var tags = new TagList
{
{ "tenant", NormalizeTenant(tenant) },
{ "effect", NormalizeTag(effectType) },
};
ExceptionApplicationsCounter.Add(1, tags);
}
/// <summary>
/// Records latency attributed to exception application during evaluation.
/// </summary>
public static void RecordExceptionApplicationLatency(double seconds, string tenant, string effectType)
{
var tags = new TagList
{
{ "tenant", NormalizeTenant(tenant) },
{ "effect", NormalizeTag(effectType) },
};
ExceptionApplicationLatencyHistogram.Record(seconds, tags);
}
/// <summary>
/// Records an exception lifecycle event (activated, expired, revoked).
/// </summary>
public static void RecordExceptionLifecycle(string tenant, string eventType)
{
var tags = new TagList
{
{ "tenant", NormalizeTenant(tenant) },
{ "event", NormalizeTag(eventType) },
};
ExceptionLifecycleCounter.Add(1, tags);
}
#region Golden Signals - Recording Methods

View File

@@ -0,0 +1,127 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Policy.Engine.ExceptionCache;
using StellaOps.Policy.Engine.Events;
using StellaOps.Policy.Engine.Options;
using StellaOps.Policy.Engine.Storage.Mongo.Repositories;
using StellaOps.Policy.Engine.Telemetry;
namespace StellaOps.Policy.Engine.Workers;
/// <summary>
/// Executes activation/expiry flows for exceptions and emits lifecycle events.
/// Split from the hosted worker for testability.
/// </summary>
internal sealed class ExceptionLifecycleService
{
private readonly IExceptionRepository _repository;
private readonly IExceptionEventPublisher _publisher;
private readonly IOptions<PolicyEngineOptions> _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<ExceptionLifecycleService> _logger;
public ExceptionLifecycleService(
IExceptionRepository repository,
IExceptionEventPublisher publisher,
IOptions<PolicyEngineOptions> options,
TimeProvider timeProvider,
ILogger<ExceptionLifecycleService> logger)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_options = options ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task ProcessOnceAsync(CancellationToken cancellationToken)
{
var now = _timeProvider.GetUtcNow();
var lifecycle = _options.Value.ExceptionLifecycle;
var pendingActivations = await _repository
.ListExceptionsAsync(new ExceptionQueryOptions
{
Statuses = ImmutableArray.Create("approved"),
}, cancellationToken)
.ConfigureAwait(false);
pendingActivations = pendingActivations
.Where(ex => ex.EffectiveFrom is null || ex.EffectiveFrom <= now)
.Take(lifecycle.MaxBatchSize)
.ToImmutableArray();
foreach (var ex in pendingActivations)
{
var activated = await _repository.UpdateExceptionStatusAsync(
ex.TenantId, ex.Id, "active", now, cancellationToken).ConfigureAwait(false);
if (!activated)
{
continue;
}
PolicyEngineTelemetry.RecordExceptionLifecycle(ex.TenantId, "activated");
await _publisher.PublishAsync(new ExceptionEvent
{
EventType = "activated",
TenantId = ex.TenantId,
ExceptionId = ex.Id,
ExceptionName = ex.Name,
ExceptionType = ex.ExceptionType,
OccurredAt = now,
}, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Activated exception {ExceptionId} for tenant {TenantId} (effective from {EffectiveFrom:o})",
ex.Id,
ex.TenantId,
ex.EffectiveFrom);
}
var expiryWindowStart = now - lifecycle.ExpiryLookback;
var expiryWindowEnd = now + lifecycle.ExpiryHorizon;
var expiring = await _repository
.ListExceptionsAsync(new ExceptionQueryOptions
{
Statuses = ImmutableArray.Create("active"),
}, cancellationToken)
.ConfigureAwait(false);
expiring = expiring
.Where(ex => ex.ExpiresAt is not null && ex.ExpiresAt >= expiryWindowStart && ex.ExpiresAt <= expiryWindowEnd)
.Take(lifecycle.MaxBatchSize)
.ToImmutableArray();
foreach (var ex in expiring)
{
var expired = await _repository.UpdateExceptionStatusAsync(
ex.TenantId, ex.Id, "expired", now, cancellationToken).ConfigureAwait(false);
if (!expired)
{
continue;
}
PolicyEngineTelemetry.RecordExceptionLifecycle(ex.TenantId, "expired");
await _publisher.PublishAsync(new ExceptionEvent
{
EventType = "expired",
TenantId = ex.TenantId,
ExceptionId = ex.Id,
ExceptionName = ex.Name,
ExceptionType = ex.ExceptionType,
OccurredAt = now,
}, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Expired exception {ExceptionId} for tenant {TenantId} at {ExpiresAt:o}",
ex.Id,
ex.TenantId,
ex.ExpiresAt);
}
}
}

View File

@@ -0,0 +1,51 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using StellaOps.Policy.Engine.Options;
namespace StellaOps.Policy.Engine.Workers;
/// <summary>
/// Hosted service that periodically runs exception activation/expiry checks.
/// </summary>
internal sealed class ExceptionLifecycleWorker : BackgroundService
{
private readonly ExceptionLifecycleService _service;
private readonly PolicyEngineExceptionLifecycleOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<ExceptionLifecycleWorker> _logger;
public ExceptionLifecycleWorker(
ExceptionLifecycleService service,
PolicyEngineExceptionLifecycleOptions options,
TimeProvider timeProvider,
ILogger<ExceptionLifecycleWorker> logger)
{
_service = service ?? throw new ArgumentNullException(nameof(service));
_options = options ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Starting exception lifecycle worker (interval {Interval}s)", _options.PollIntervalSeconds);
while (!stoppingToken.IsCancellationRequested)
{
try
{
await _service.ProcessOnceAsync(stoppingToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "Exception lifecycle worker iteration failed");
}
await Task.Delay(_options.PollInterval, stoppingToken).ConfigureAwait(false);
}
}
}

View File

@@ -61,6 +61,18 @@ public sealed record CvssThreatMetrics
/// <summary>Exploit Maturity (E) - Optional, defaults to Not Defined.</summary>
[JsonPropertyName("e")]
public ExploitMaturity ExploitMaturity { get; init; } = ExploitMaturity.NotDefined;
/// <summary>When the threat signal was last observed (UTC).</summary>
[JsonPropertyName("observedAt")]
public DateTimeOffset? ObservedAt { get; init; }
/// <summary>When this threat signal should expire.</summary>
[JsonPropertyName("expiresAt")]
public DateTimeOffset? ExpiresAt { get; init; }
/// <summary>Source of threat intelligence (kev, epss, internal).</summary>
[JsonPropertyName("source")]
public string? Source { get; init; }
}
/// <summary>

View File

@@ -85,6 +85,10 @@ public sealed record CvssScoreReceipt
[JsonPropertyName("evidence")]
public ImmutableList<CvssEvidenceItem> Evidence { get; init; } = [];
/// <summary>Export hash for deterministic exports (JSON/PDF).</summary>
[JsonPropertyName("exportHash")]
public string? ExportHash { get; init; }
/// <summary>DSSE attestation envelope references, if signed.</summary>
[JsonPropertyName("attestationRefs")]
public ImmutableList<string> AttestationRefs { get; init; } = [];
@@ -101,6 +105,10 @@ public sealed record CvssScoreReceipt
[JsonPropertyName("amendsReceiptId")]
public string? AmendsReceiptId { get; init; }
/// <summary>Supersedes prior receipt when policy changes or replays occur.</summary>
[JsonPropertyName("supersedesReceiptId")]
public string? SupersedesReceiptId { get; init; }
/// <summary>Whether this receipt is the current active version.</summary>
[JsonPropertyName("isActive")]
public bool IsActive { get; init; } = true;
@@ -224,6 +232,22 @@ public sealed record CvssEvidenceItem
/// <summary>Whether this evidence is from the vendor/authority.</summary>
[JsonPropertyName("isAuthoritative")]
public bool IsAuthoritative { get; init; }
/// <summary>Retention class (short, standard, long) for this evidence.</summary>
[JsonPropertyName("retentionClass")]
public string? RetentionClass { get; init; }
/// <summary>DSSE reference for the evidence, if signed.</summary>
[JsonPropertyName("dsseRef")]
public string? DsseRef { get; init; }
/// <summary>Whether the evidence has been redacted to remove sensitive data.</summary>
[JsonPropertyName("isRedacted")]
public bool? IsRedacted { get; init; }
/// <summary>When the evidence hash was last verified against CAS.</summary>
[JsonPropertyName("verifiedAt")]
public DateTimeOffset? VerifiedAt { get; init; }
}
/// <summary>

View File

@@ -0,0 +1,66 @@
using System.Globalization;
namespace StellaOps.Policy.Scoring.Engine;
/// <summary>
/// Deterministic interoperability helpers between CVSS v3.1 and v4.0 vectors.
/// Covers common base metrics; fields without equivalents are omitted.
/// </summary>
public static class CvssVectorInterop
{
private static readonly IReadOnlyDictionary<string, string> V31ToV4Map = new Dictionary<string, string>(StringComparer.Ordinal)
{
["AV:N"] = "AV:N",
["AV:A"] = "AV:A",
["AV:L"] = "AV:L",
["AV:P"] = "AV:P",
["AC:L"] = "AC:L",
["AC:H"] = "AC:H",
["PR:N"] = "PR:N",
["PR:L"] = "PR:L",
["PR:H"] = "PR:H",
["UI:N"] = "UI:N",
["UI:R"] = "UI:R",
["S:U"] = "VC:H,VI:H,VA:H",
["S:C"] = "VC:H,VI:H,VA:H",
["C:H"] = "VC:H",
["C:L"] = "VC:L",
["I:H"] = "VI:H",
["I:L"] = "VI:L",
["A:H"] = "VA:H",
["A:L"] = "VA:L"
};
/// <summary>
/// Converts a CVSS v3.1 base vector into an approximate CVSS v4.0 base vector.
/// Outputs only base metrics; threat/environmental must be provided separately.
/// </summary>
public static string ConvertV31ToV4(string v31Vector)
{
if (string.IsNullOrWhiteSpace(v31Vector))
{
throw new ArgumentException("Vector cannot be null or empty", nameof(v31Vector));
}
var parts = v31Vector.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Where(p => p.Contains(':'))
.ToList();
var mapped = new List<string> { "CVSS:4.0" };
foreach (var part in parts)
{
if (V31ToV4Map.TryGetValue(part, out var v4))
{
mapped.AddRange(v4.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
}
}
var deduped = mapped.Distinct(StringComparer.Ordinal)
.OrderBy(p => p == "CVSS:4.0" ? 0 : 1)
.ThenBy(p => p, StringComparer.Ordinal)
.ToList();
return string.Join('/', deduped);
}
}

View File

@@ -116,7 +116,9 @@ public sealed class ReceiptBuilder : IReceiptBuilder
}),
AmendsReceiptId = null,
IsActive = true,
SupersededReason = null
SupersedesReceiptId = null,
SupersededReason = null,
ExportHash = null
};
if (request.SigningKey is not null)
@@ -166,57 +168,24 @@ public sealed class ReceiptBuilder : IReceiptBuilder
string vector,
ImmutableList<CvssEvidenceItem> evidence)
{
using var stream = new MemoryStream();
using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions
using var doc = JsonDocument.Parse(JsonSerializer.Serialize(new
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = false
});
vulnerabilityId = request.VulnerabilityId,
tenantId = request.TenantId,
policyId = policyRef.PolicyId,
policyVersion = policyRef.Version,
policyHash = policyRef.Hash,
vector,
baseMetrics = request.BaseMetrics,
threatMetrics = request.ThreatMetrics,
environmentalMetrics = request.EnvironmentalMetrics,
supplementalMetrics = request.SupplementalMetrics,
scores,
evidence
}, SerializerOptions));
writer.WriteStartObject();
writer.WriteString("vulnerabilityId", request.VulnerabilityId);
writer.WriteString("tenantId", request.TenantId);
writer.WriteString("policyId", policyRef.PolicyId);
writer.WriteString("policyVersion", policyRef.Version);
writer.WriteString("policyHash", policyRef.Hash);
writer.WriteString("vector", vector);
writer.WritePropertyName("baseMetrics");
WriteCanonical(JsonSerializer.SerializeToElement(request.BaseMetrics, SerializerOptions), writer);
writer.WritePropertyName("threatMetrics");
if (request.ThreatMetrics is not null)
WriteCanonical(JsonSerializer.SerializeToElement(request.ThreatMetrics, SerializerOptions), writer);
else
writer.WriteNullValue();
writer.WritePropertyName("environmentalMetrics");
if (request.EnvironmentalMetrics is not null)
WriteCanonical(JsonSerializer.SerializeToElement(request.EnvironmentalMetrics, SerializerOptions), writer);
else
writer.WriteNullValue();
writer.WritePropertyName("supplementalMetrics");
if (request.SupplementalMetrics is not null)
WriteCanonical(JsonSerializer.SerializeToElement(request.SupplementalMetrics, SerializerOptions), writer);
else
writer.WriteNullValue();
writer.WritePropertyName("scores");
WriteCanonical(JsonSerializer.SerializeToElement(scores, SerializerOptions), writer);
writer.WritePropertyName("evidence");
writer.WriteStartArray();
foreach (var ev in evidence)
{
WriteCanonical(JsonSerializer.SerializeToElement(ev, SerializerOptions), writer);
}
writer.WriteEndArray();
writer.WriteEndObject();
writer.Flush();
var hash = SHA256.HashData(stream.ToArray());
var canonicalBytes = ReceiptCanonicalizer.ToCanonicalBytes(doc.RootElement);
var hash = SHA256.HashData(canonicalBytes);
return Convert.ToHexString(hash).ToLowerInvariant();
}

View File

@@ -0,0 +1,84 @@
using System.Globalization;
using System.Text.Encodings.Web;
using System.Text.Json;
namespace StellaOps.Policy.Scoring.Receipts;
/// <summary>
/// Provides deterministic JSON canonicalization for receipt hashing.
/// Keys are sorted, numbers use invariant culture, and timestamps are formatted as ISO 8601 (O).
/// </summary>
internal static class ReceiptCanonicalizer
{
private static readonly JsonWriterOptions WriterOptions = new()
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
Indented = false
};
public static byte[] ToCanonicalBytes(JsonElement element)
{
using var stream = new MemoryStream();
using var writer = new Utf8JsonWriter(stream, WriterOptions);
Write(element, writer);
writer.Flush();
return stream.ToArray();
}
public static void Write(JsonElement element, Utf8JsonWriter writer)
{
switch (element.ValueKind)
{
case JsonValueKind.Object:
writer.WriteStartObject();
foreach (var prop in element.EnumerateObject().OrderBy(p => p.Name, StringComparer.Ordinal))
{
writer.WritePropertyName(prop.Name);
Write(prop.Value, writer);
}
writer.WriteEndObject();
break;
case JsonValueKind.Array:
writer.WriteStartArray();
foreach (var item in element.EnumerateArray())
{
Write(item, writer);
}
writer.WriteEndArray();
break;
case JsonValueKind.String:
// If the value looks like a timestamp, normalize to ISO 8601 round-trip
if (DateTimeOffset.TryParse(element.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var dto))
{
writer.WriteStringValue(dto.ToUniversalTime().ToString("O"));
}
else
{
writer.WriteStringValue(element.GetString());
}
break;
case JsonValueKind.Number:
if (element.TryGetDouble(out var dbl))
{
writer.WriteRawValue(dbl.ToString("0.################", CultureInfo.InvariantCulture), skipInputValidation: true);
}
else
{
writer.WriteRawValue(element.GetRawText(), skipInputValidation: true);
}
break;
case JsonValueKind.True:
writer.WriteBooleanValue(true);
break;
case JsonValueKind.False:
writer.WriteBooleanValue(false);
break;
case JsonValueKind.Null:
case JsonValueKind.Undefined:
writer.WriteNullValue();
break;
default:
throw new InvalidOperationException($"Unsupported JSON value kind: {element.ValueKind}");
}
}
}

View File

@@ -57,6 +57,8 @@
"supplementalMetrics": {
"$ref": "#/$defs/supplementalMetrics"
},
"exportHash": { "type": "string" },
"supersedesReceiptId": { "type": "string" },
"scores": {
"$ref": "#/$defs/scores"
},
@@ -120,7 +122,10 @@
"threatMetrics": {
"type": "object",
"properties": {
"e": { "type": "string", "enum": ["NotDefined", "Attacked", "ProofOfConcept", "Unreported"] }
"e": { "type": "string", "enum": ["NotDefined", "Attacked", "ProofOfConcept", "Unreported"] },
"observedAt": { "type": "string", "format": "date-time" },
"expiresAt": { "type": "string", "format": "date-time" },
"source": { "type": "string" }
}
},
"environmentalMetrics": {
@@ -184,7 +189,11 @@
"description": { "type": "string" },
"collectedAt": { "type": "string", "format": "date-time" },
"source": { "type": "string" },
"isAuthoritative": { "type": "boolean", "default": false }
"isAuthoritative": { "type": "boolean", "default": false },
"retentionClass": { "type": "string" },
"dsseRef": { "type": "string" },
"isRedacted": { "type": "boolean" },
"verifiedAt": { "type": "string", "format": "date-time" }
}
},
"historyEntry": {

View File

@@ -0,0 +1,91 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Policy;
using StellaOps.Policy.Engine.BatchEvaluation;
using StellaOps.Policy.Engine.Services;
using Xunit;
namespace StellaOps.Policy.Engine.Tests.BatchEvaluation;
public sealed class BatchEvaluationMapperTests
{
[Fact]
public void Validate_Fails_WhenTimestampMissing()
{
var request = new BatchEvaluationRequestDto(
TenantId: "acme",
Items: new[]
{
new BatchEvaluationItemDto(
PackId: "pack-1",
Version: 1,
SubjectPurl: "pkg:npm/lodash@4.17.21",
AdvisoryId: "ADV-1",
Severity: new EvaluationSeverityDto("high", 7.5m),
Advisory: new AdvisoryDto(new Dictionary<string, string>(), "nvd"),
Vex: new VexEvidenceDto(Array.Empty<VexStatementDto>()),
Sbom: new SbomDto(Array.Empty<string>()),
Exceptions: new ExceptionsDto(),
Reachability: new ReachabilityDto("unknown"),
EvaluationTimestamp: null)
});
var ok = BatchEvaluationValidator.TryValidate(request, out var error);
Assert.False(ok);
Assert.Contains("evaluationTimestamp", error, StringComparison.OrdinalIgnoreCase);
}
[Fact]
public void Mapper_Produces_RuntimeRequest_WithSuppliedValues()
{
var item = new BatchEvaluationItemDto(
PackId: "pack-1",
Version: 2,
SubjectPurl: "pkg:npm/foo@1.0.0",
AdvisoryId: "ADV-1",
Severity: new EvaluationSeverityDto("high", 8.0m),
Advisory: new AdvisoryDto(new Dictionary<string, string>
{
["cve"] = "CVE-2025-0001"
}, "nvd"),
Vex: new VexEvidenceDto(new[]
{
new VexStatementDto("not_affected", "vendor_confirmed", "stmt-1", new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero))
}),
Sbom: new SbomDto(
Tags: new[] { "runtime", "server" },
Components: new[]
{
new ComponentDto("foo", "1.0.0", "npm", "pkg:npm/foo@1.0.0")
}),
Exceptions: new ExceptionsDto(
Effects: new Dictionary<string, PolicyExceptionEffect>(),
Instances: new[]
{
new ExceptionInstanceDto(
Id: "ex-1",
EffectId: "suppress",
Scope: new ExceptionScopeDto(
RuleNames: new[] { "rule-1" },
Severities: new[] { "high" }),
CreatedAt: new DateTimeOffset(2025, 1, 2, 0, 0, 0, TimeSpan.Zero))
}),
Reachability: new ReachabilityDto("reachable", 0.9m, 0.8m, HasRuntimeEvidence: true, Source: "scanner", Method: "static", EvidenceRef: "evidence-1"),
EvaluationTimestamp: new DateTimeOffset(2025, 1, 3, 0, 0, 0, TimeSpan.Zero),
BypassCache: false);
var runtimeRequests = BatchEvaluationMapper.ToRuntimeRequests("acme", new[] { item });
var runtime = Assert.Single(runtimeRequests);
Assert.Equal("acme", runtime.TenantId);
Assert.Equal("pack-1", runtime.PackId);
Assert.Equal("pkg:npm/foo@1.0.0", runtime.SubjectPurl);
Assert.Equal(new DateTimeOffset(2025, 1, 3, 0, 0, 0, TimeSpan.Zero), runtime.EvaluationTimestamp);
Assert.Equal("reachable", runtime.Reachability.State);
Assert.True(runtime.Reachability.HasRuntimeEvidence);
Assert.Equal("scanner", runtime.Reachability.Source);
Assert.Equal("high", runtime.Severity.Normalized);
}
}

View File

@@ -5,6 +5,7 @@ using StellaOps.Policy.Engine.Caching;
using StellaOps.Policy.Engine.Compilation;
using StellaOps.Policy.Engine.Domain;
using StellaOps.Policy.Engine.Evaluation;
using StellaOps.Policy.Engine.ReachabilityFacts;
using StellaOps.Policy.Engine.Options;
using StellaOps.Policy.Engine.Services;
using StellaOps.PolicyDsl;
@@ -180,6 +181,55 @@ public sealed class PolicyRuntimeEvaluationServiceTests
Assert.NotEqual(response1.CorrelationId, response2.CorrelationId);
}
[Fact]
public async Task EvaluateAsync_EnrichesReachabilityFromFacts()
{
const string policy = """
policy "Reachability policy" syntax "stella-dsl@1" {
rule reachable_then_warn priority 5 {
when reachability.state == "reachable"
then status := "warn"
because "reachable path detected"
}
rule default priority 100 {
when true
then status := "affected"
because "default"
}
}
""";
var harness = CreateHarness();
await harness.StoreTestPolicyAsync("pack-2", 1, policy);
var fact = new ReachabilityFact
{
Id = "fact-1",
TenantId = "tenant-1",
ComponentPurl = "pkg:npm/lodash@4.17.21",
AdvisoryId = "CVE-2024-0001",
State = ReachabilityState.Reachable,
Confidence = 0.92m,
Score = 0.85m,
HasRuntimeEvidence = true,
Source = "graph-analyzer",
Method = AnalysisMethod.Hybrid,
EvidenceRef = "evidence/callgraph.json",
ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
ExpiresAt = null,
Metadata = new Dictionary<string, object?>()
};
await harness.ReachabilityStore.SaveAsync(fact, CancellationToken.None);
var request = CreateRequest("pack-2", 1, severity: "Low");
var response = await harness.Service.EvaluateAsync(request, CancellationToken.None);
Assert.Equal("warn", response.Status);
}
private static RuntimeEvaluationRequest CreateRequest(
string packId,
int version,
@@ -213,16 +263,28 @@ public sealed class PolicyRuntimeEvaluationServiceTests
var cache = new InMemoryPolicyEvaluationCache(cacheLogger, TimeProvider.System, options);
var evaluator = new PolicyEvaluator();
var reachabilityStore = new InMemoryReachabilityFactsStore(TimeProvider.System);
var reachabilityCache = new InMemoryReachabilityFactsOverlayCache(
NullLogger<InMemoryReachabilityFactsOverlayCache>.Instance,
TimeProvider.System,
Microsoft.Extensions.Options.Options.Create(new PolicyEngineOptions()));
var reachabilityService = new ReachabilityFactsJoiningService(
reachabilityStore,
reachabilityCache,
NullLogger<ReachabilityFactsJoiningService>.Instance,
TimeProvider.System);
var compilationService = CreateCompilationService();
var service = new PolicyRuntimeEvaluationService(
repository,
cache,
evaluator,
reachabilityService,
TimeProvider.System,
serviceLogger);
return new TestHarness(service, repository, compilationService);
return new TestHarness(service, repository, compilationService, reachabilityStore);
}
private static PolicyCompilationService CreateCompilationService()
@@ -238,7 +300,8 @@ public sealed class PolicyRuntimeEvaluationServiceTests
private sealed record TestHarness(
PolicyRuntimeEvaluationService Service,
InMemoryPolicyPackRepository Repository,
PolicyCompilationService CompilationService)
PolicyCompilationService CompilationService,
InMemoryReachabilityFactsStore ReachabilityStore)
{
public async Task StoreTestPolicyAsync(string packId, int version, string dsl)
{

View File

@@ -0,0 +1,88 @@
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Policy.Engine.Events;
using StellaOps.Policy.Engine.Options;
using StellaOps.Policy.Engine.Storage.InMemory;
using StellaOps.Policy.Engine.Storage.Mongo.Documents;
using StellaOps.Policy.Engine.Workers;
using StellaOps.Policy.Engine.ExceptionCache;
using Xunit;
namespace StellaOps.Policy.Engine.Tests.Workers;
public sealed class ExceptionLifecycleServiceTests
{
[Fact]
public async Task Activates_pending_exceptions_and_publishes_event()
{
var time = new FakeTimeProvider(new DateTimeOffset(2025, 12, 1, 12, 0, 0, TimeSpan.Zero));
var repo = new InMemoryExceptionRepository();
await repo.CreateExceptionAsync(new PolicyExceptionDocument
{
Id = "exc-1",
TenantId = "tenant-a",
Status = "approved",
Name = "Test exception",
EffectiveFrom = time.GetUtcNow().AddMinutes(-1),
}, CancellationToken.None);
var publisher = new RecordingPublisher();
var options = Microsoft.Extensions.Options.Options.Create(new PolicyEngineOptions());
var service = new ExceptionLifecycleService(
repo,
publisher,
options,
time,
NullLogger<ExceptionLifecycleService>.Instance);
await service.ProcessOnceAsync(CancellationToken.None);
var updated = await repo.GetExceptionAsync("tenant-a", "exc-1", CancellationToken.None);
updated!.Status.Should().Be("active");
publisher.Events.Should().ContainSingle(e => e.EventType == "activated" && e.ExceptionId == "exc-1");
}
[Fact]
public async Task Expires_active_exceptions_and_publishes_event()
{
var time = new FakeTimeProvider(new DateTimeOffset(2025, 12, 1, 12, 0, 0, TimeSpan.Zero));
var repo = new InMemoryExceptionRepository();
await repo.CreateExceptionAsync(new PolicyExceptionDocument
{
Id = "exc-2",
TenantId = "tenant-b",
Status = "active",
Name = "Expiring exception",
ExpiresAt = time.GetUtcNow().AddMinutes(-1),
}, CancellationToken.None);
var publisher = new RecordingPublisher();
var options = Microsoft.Extensions.Options.Options.Create(new PolicyEngineOptions());
var service = new ExceptionLifecycleService(
repo,
publisher,
options,
time,
NullLogger<ExceptionLifecycleService>.Instance);
await service.ProcessOnceAsync(CancellationToken.None);
var updated = await repo.GetExceptionAsync("tenant-b", "exc-2", CancellationToken.None);
updated!.Status.Should().Be("expired");
publisher.Events.Should().ContainSingle(e => e.EventType == "expired" && e.ExceptionId == "exc-2");
}
private sealed class RecordingPublisher : IExceptionEventPublisher
{
public List<ExceptionEvent> Events { get; } = new();
public Task PublishAsync(ExceptionEvent exceptionEvent, CancellationToken cancellationToken = default)
{
Events.Add(exceptionEvent);
return Task.CompletedTask;
}
}
}

View File

@@ -0,0 +1,20 @@
using FluentAssertions;
using StellaOps.Policy.Scoring.Engine;
using Xunit;
namespace StellaOps.Policy.Scoring.Tests;
public class CvssVectorInteropTests
{
[Theory]
[InlineData("CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", "CVSS:4.0/AV:N/AC:L/PR:N/UI:N/VC:H/VI:H/VA:H")]
[InlineData("CVSS:3.1/AV:L/AC:H/PR:H/UI:R/S:U/C:L/I:L/A:L", "CVSS:4.0/AV:L/AC:H/PR:H/UI:R/VC:L/VI:L/VA:L")]
public void ConvertV31ToV4_ProducesDeterministicVector(string v31, string expectedPrefix)
{
var v4 = CvssVectorInterop.ConvertV31ToV4(v31);
v4.Should().StartWith(expectedPrefix);
// determinism: same input produces identical output
CvssVectorInterop.ConvertV31ToV4(v31).Should().Be(v4);
}
}

View File

@@ -1,5 +1,6 @@
using System.Collections.Immutable;
using FluentAssertions;
using System.Linq;
using StellaOps.Attestor.Envelope;
using StellaOps.Policy.Scoring.Engine;
using StellaOps.Policy.Scoring.Receipts;
@@ -71,6 +72,74 @@ public sealed class ReceiptBuilderTests
_repository.Contains(receipt1.ReceiptId).Should().BeTrue();
}
[Fact]
public async Task CreateAsync_InputHashIgnoresPropertyOrder()
{
var policy = new CvssPolicy
{
PolicyId = "default",
Version = "1.0.0",
Name = "Default",
EffectiveFrom = new DateTimeOffset(2025, 01, 01, 0, 0, 0, TimeSpan.Zero),
Hash = "abc123",
};
var baseMetrics = new CvssBaseMetrics
{
AttackVector = AttackVector.Network,
AttackComplexity = AttackComplexity.Low,
AttackRequirements = AttackRequirements.None,
PrivilegesRequired = PrivilegesRequired.None,
UserInteraction = UserInteraction.None,
VulnerableSystemConfidentiality = ImpactMetricValue.High,
VulnerableSystemIntegrity = ImpactMetricValue.High,
VulnerableSystemAvailability = ImpactMetricValue.High,
SubsequentSystemConfidentiality = ImpactMetricValue.High,
SubsequentSystemIntegrity = ImpactMetricValue.High,
SubsequentSystemAvailability = ImpactMetricValue.High
};
var evidence1 = ImmutableList<CvssEvidenceItem>.Empty.Add(new CvssEvidenceItem
{
Type = "scan",
Uri = "sha256:1",
Description = "First",
IsAuthoritative = false
}).Add(new CvssEvidenceItem
{
Type = "advisory",
Uri = "sha256:2",
Description = "Second",
IsAuthoritative = true
});
var evidence2 = evidence1.Reverse().ToImmutableList();
var builder = new ReceiptBuilder(_engine, _repository);
var r1 = await builder.CreateAsync(new CreateReceiptRequest
{
VulnerabilityId = "CVE-2025-1111",
TenantId = "t",
CreatedBy = "u",
Policy = policy,
BaseMetrics = baseMetrics,
Evidence = evidence1
});
var r2 = await builder.CreateAsync(new CreateReceiptRequest
{
VulnerabilityId = "CVE-2025-1111",
TenantId = "t",
CreatedBy = "u",
Policy = policy,
BaseMetrics = baseMetrics,
Evidence = evidence2
});
r1.InputHash.Should().Be(r2.InputHash);
}
[Fact]
public async Task CreateAsync_WithSigningKey_AttachesDsseReference()
{

View File

@@ -7,4 +7,5 @@ public sealed record EntryTraceResponse(
string ImageDigest,
DateTimeOffset GeneratedAt,
EntryTraceGraph Graph,
IReadOnlyList<string> Ndjson);
IReadOnlyList<string> Ndjson,
EntryTracePlan? BestPlan = null);

View File

@@ -424,12 +424,18 @@ internal static class ScanEndpoints
}
}
var bestPlan = result.Graph.Plans
.OrderByDescending(p => p.Confidence)
.ThenBy(p => p.TerminalPath, StringComparer.Ordinal)
.FirstOrDefault();
var response = new EntryTraceResponse(
result.ScanId,
result.ImageDigest,
result.GeneratedAtUtc,
result.Graph,
result.Ndjson);
result.Ndjson,
bestPlan);
return Json(response, StatusCodes.Status200OK);
}

View File

@@ -140,8 +140,15 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
{
var payloads = new List<SurfaceManifestPayload>();
EntryTracePlan? bestPlan = null;
if (context.Analysis.TryGet<EntryTraceGraph>(ScanAnalysisKeys.EntryTraceGraph, out var graph) && graph is not null)
{
bestPlan = graph.Plans
.OrderByDescending(p => p.Confidence)
.ThenBy(p => p.TerminalPath, StringComparer.Ordinal)
.FirstOrDefault();
var graphJson = EntryTraceGraphSerializer.Serialize(graph);
payloads.Add(new SurfaceManifestPayload(
ArtifactDocumentType.SurfaceEntryTrace,
@@ -149,12 +156,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
Kind: "entrytrace.graph",
MediaType: "application/json",
Content: Encoding.UTF8.GetBytes(graphJson),
Metadata: new Dictionary<string, string>
{
["artifact"] = "entrytrace.graph",
["nodes"] = graph.Nodes.Length.ToString(CultureInfoInvariant),
["edges"] = graph.Edges.Length.ToString(CultureInfoInvariant)
}));
Metadata: BuildEntryTraceMetadata(graph, bestPlan)));
}
if (context.Analysis.TryGet(ScanAnalysisKeys.EntryTraceNdjson, out ImmutableArray<string> ndjson) && !ndjson.IsDefaultOrEmpty)
@@ -174,7 +176,8 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
ArtifactDocumentFormat.EntryTraceNdjson,
Kind: "entrytrace.ndjson",
MediaType: "application/x-ndjson",
Content: Encoding.UTF8.GetBytes(builder.ToString())));
Content: Encoding.UTF8.GetBytes(builder.ToString()),
Metadata: BuildEntryTraceMetadata(graph: null, bestPlan)));
}
var fragments = context.Analysis.GetLayerFragments();
@@ -330,6 +333,30 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
? value.Trim()
: null;
private static Dictionary<string, string> BuildEntryTraceMetadata(EntryTraceGraph? graph, EntryTracePlan? bestPlan)
{
var metadata = new Dictionary<string, string>
{
["artifact"] = graph is null ? "entrytrace.ndjson" : "entrytrace.graph"
};
if (graph is not null)
{
metadata["nodes"] = graph.Nodes.Length.ToString(CultureInfoInvariant);
metadata["edges"] = graph.Edges.Length.ToString(CultureInfoInvariant);
}
if (bestPlan is not null)
{
metadata["best_terminal"] = bestPlan.Value.TerminalPath;
metadata["best_confidence"] = bestPlan.Value.Confidence.ToString("F4", CultureInfoInvariant);
metadata["best_user"] = bestPlan.Value.User;
metadata["best_workdir"] = bestPlan.Value.WorkingDirectory;
}
return metadata;
}
private async Task PersistRubyPackagesAsync(ScanJobContext context, CancellationToken cancellationToken)
{
if (!context.Analysis.TryGet<ReadOnlyDictionary<string, LanguageAnalyzerResult>>(ScanAnalysisKeys.LanguageAnalyzerResults, out var results))

View File

@@ -0,0 +1,119 @@
using System.Text.RegularExpressions;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal static class NodeEnvironmentScanner
{
private static readonly Regex EnvAssign = new("^\s*(ENV|ARG)\s+NODE_OPTIONS\s*(=|\s)(?<value>.+)$", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static IReadOnlyList<LanguageComponentRecord> Scan(LanguageAnalyzerContext context, IReadOnlyList<string> sourceRoots, CancellationToken cancellationToken)
{
var warnings = new List<LanguageComponentRecord>();
foreach (var root in sourceRoots)
{
cancellationToken.ThrowIfCancellationRequested();
var dockerfile = Path.Combine(root, "Dockerfile");
if (File.Exists(dockerfile))
{
warnings.AddRange(ScanDockerfile(context, dockerfile));
}
var envFile = Path.Combine(root, ".env");
if (File.Exists(envFile))
{
warnings.AddRange(ScanEnvFile(context, envFile));
}
}
return warnings
.OrderBy(static r => r.ComponentKey, StringComparer.Ordinal)
.ToArray();
}
private static IEnumerable<LanguageComponentRecord> ScanDockerfile(LanguageAnalyzerContext context, string dockerfile)
{
try
{
var lines = File.ReadAllLines(dockerfile);
for (var i = 0; i < lines.Length; i++)
{
var match = EnvAssign.Match(lines[i]);
if (!match.Success)
{
continue;
}
var value = match.Groups["value"].Value.Trim().Trim('"', '\'');
yield return BuildWarning(context, dockerfile, i + 1, value, source: "Dockerfile", reason: "NODE_OPTIONS");
}
}
catch (IOException)
{
yield break;
}
}
private static IEnumerable<LanguageComponentRecord> ScanEnvFile(LanguageAnalyzerContext context, string envFile)
{
try
{
var lines = File.ReadAllLines(envFile);
for (var i = 0; i < lines.Length; i++)
{
var line = lines[i];
if (!line.Contains("NODE_OPTIONS", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var parts = line.Split('=', 2);
if (parts.Length != 2)
{
continue;
}
var value = parts[1].Trim().Trim('"', '\'');
yield return BuildWarning(context, envFile, i + 1, value, source: ".env", reason: "NODE_OPTIONS");
}
}
catch (IOException)
{
yield break;
}
}
private static LanguageComponentRecord BuildWarning(LanguageAnalyzerContext context, string filePath, int lineNumber, string value, string source, string reason)
{
var locator = context.GetRelativePath(filePath).Replace(Path.DirectorySeparatorChar, '/');
var metadata = new List<KeyValuePair<string, string?>>
{
new("source", source),
new("locator", string.Concat(locator, "#", lineNumber.ToString(CultureInfo.InvariantCulture))),
new("reason", reason),
new("value", value)
};
var evidence = new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.Metadata,
"node.env",
string.Concat(locator, "#", lineNumber.ToString(CultureInfo.InvariantCulture)),
value,
null)
};
return LanguageComponentRecord.FromExplicitKey(
analyzerId: "node",
componentKey: string.Concat("warning:node-options:", locator, "#", lineNumber.ToString(CultureInfo.InvariantCulture)),
purl: null,
name: "NODE_OPTIONS warning",
version: null,
type: "node:warning",
metadata: metadata,
evidence: evidence,
usedByEntrypoint: false);
}
}

View File

@@ -4,7 +4,8 @@ internal sealed record NodeImportEdge(
string SourceFile,
string TargetSpecifier,
string Kind,
string Evidence)
string Evidence,
string Confidence)
{
public string ComparisonKey => string.Concat(SourceFile, "|", TargetSpecifier, "|", Kind);
}

View File

@@ -0,0 +1,8 @@
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal sealed record NodeImportResolution(
string SourceFile,
string Specifier,
string ResolvedPath,
string ResolutionType,
string Confidence);

View File

@@ -1,3 +1,7 @@
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using Esprima;
using Esprima.Ast;
using EsprimaNode = Esprima.Ast.Node;
@@ -6,7 +10,9 @@ namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal static class NodeImportWalker
{
public static IReadOnlyList<NodeImportEdge> AnalyzeImports(string sourcePath, string content)
private const int MaxSourceMapBytes = 1_048_576; // 1 MiB safety cap
public static IReadOnlyList<NodeImportEdge> AnalyzeImports(string rootPath, string sourcePath, string content)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourcePath);
if (content is null)
@@ -14,36 +20,60 @@ internal static class NodeImportWalker
return Array.Empty<NodeImportEdge>();
}
var edges = new List<NodeImportEdge>();
AnalyzeInternal(rootPath, sourcePath, content, allowSourceMap: true, edges);
return edges.Count == 0
? Array.Empty<NodeImportEdge>()
: edges.OrderBy(static e => e.ComparisonKey, StringComparer.Ordinal).ToArray();
}
private static void AnalyzeInternal(string rootPath, string sourcePath, string content, bool allowSourceMap, List<NodeImportEdge> edges)
{
Script script;
try
{
var parser = new JavaScriptParser();
script = parser.ParseScript(content, sourcePath, true);
var parser = new JavaScriptParser(new ParserOptions { Tolerant = true });
script = parser.ParseScript(content, sourcePath, strict: false);
}
catch (ParserException)
{
return Array.Empty<NodeImportEdge>();
script = null!;
}
var edges = new List<NodeImportEdge>();
Walk(script, sourcePath, edges);
return edges.Count == 0
? Array.Empty<NodeImportEdge>()
: edges.OrderBy(e => e.ComparisonKey, StringComparer.Ordinal).ToArray();
if (script is not null)
{
Walk(script, sourcePath, edges);
}
if (allowSourceMap)
{
TryAnalyzeSourceMap(rootPath, sourcePath, content, edges);
}
}
private static void Walk(EsprimaNode node, string sourcePath, List<NodeImportEdge> edges)
{
switch (node)
{
case ImportDeclaration importDecl when !string.IsNullOrWhiteSpace(importDecl.Source?.StringValue):
edges.Add(new NodeImportEdge(sourcePath, importDecl.Source.StringValue!, "import", string.Empty));
case ImportDeclaration importDecl:
if (TryGetLiteral(importDecl.Source, out var importTarget, out var importConfidence, out var importEvidence))
{
AddEdge(edges, sourcePath, importTarget!, "import", importEvidence, importConfidence);
}
break;
case CallExpression call when IsRequire(call) && call.Arguments.FirstOrDefault() is Literal { Value: string target }:
edges.Add(new NodeImportEdge(sourcePath, target, "require", string.Empty));
case CallExpression call when IsRequire(call):
if (call.Arguments.FirstOrDefault() is Expression requireArg && TryRenderSpecifier(requireArg, out var requireTarget, out var requireConfidence, out var requireEvidence))
{
AddEdge(edges, sourcePath, requireTarget!, "require", requireEvidence, requireConfidence);
}
break;
case ImportExpression importExp when importExp.Source is Literal { Value: string importTarget }:
edges.Add(new NodeImportEdge(sourcePath, importTarget, "import()", string.Empty));
case ImportExpression importExp:
if (importExp.Source is Expression expr && TryRenderSpecifier(expr, out var importExprTarget, out var importExprConfidence, out var importExprEvidence))
{
AddEdge(edges, sourcePath, importExprTarget!, "import()", importExprEvidence, importExprConfidence);
}
break;
}
@@ -53,9 +83,199 @@ internal static class NodeImportWalker
}
}
private static bool TryRenderSpecifier(Expression expression, out string? specifier, out string confidence, out string evidence)
{
specifier = null;
evidence = string.Empty;
confidence = "low";
if (TryGetLiteral(expression, out var literalValue, out confidence, out evidence))
{
specifier = literalValue;
return true;
}
if (expression is TemplateLiteral template)
{
var raw = string.Concat(template.Quasis.Select(static q => q.Value?.Cooked ?? string.Empty));
if (template.Expressions.Count == 0)
{
specifier = raw;
confidence = "high";
evidence = "template-static";
return true;
}
specifier = raw + "${*}";
confidence = "medium";
evidence = "template-dynamic";
return true;
}
if (expression is BinaryExpression binary && binary.Operator == BinaryOperator.Plus)
{
var leftOk = TryRenderSpecifier(binary.Left, out var left, out var leftConf, out var leftEvidence);
var rightOk = TryRenderSpecifier(binary.Right, out var right, out var rightConf, out var rightEvidence);
if (leftOk || rightOk)
{
specifier = string.Concat(left ?? string.Empty, right ?? string.Empty);
var combinedLeft = leftOk ? leftConf : rightConf;
var combinedRight = rightOk ? rightConf : leftConf;
confidence = CombineConfidence(combinedLeft, combinedRight);
evidence = string.Join(';', new[] { leftEvidence, rightEvidence }.Where(static e => !string.IsNullOrWhiteSpace(e)));
return true;
}
}
return false;
}
private static bool TryGetLiteral(EsprimaNode? node, out string? value, out string confidence, out string evidence)
{
value = null;
confidence = "low";
evidence = string.Empty;
if (node is Literal { Value: string literal })
{
value = literal;
confidence = "high";
evidence = "literal";
return true;
}
return false;
}
private static void AddEdge(List<NodeImportEdge> edges, string sourcePath, string target, string kind, string evidence, string confidence)
{
var edge = new NodeImportEdge(sourcePath, target, kind, evidence, confidence);
if (edges.Any(e => string.Equals(e.ComparisonKey, edge.ComparisonKey, StringComparison.Ordinal)))
{
return;
}
edges.Add(edge);
}
private static void TryAnalyzeSourceMap(string rootPath, string sourcePath, string content, List<NodeImportEdge> edges)
{
var mapPath = ExtractSourceMapPath(content);
if (string.IsNullOrWhiteSpace(mapPath) || mapPath.StartsWith("data:", StringComparison.OrdinalIgnoreCase))
{
return;
}
var sourceDirectory = Path.GetDirectoryName(Path.Combine(rootPath, sourcePath.Replace('/', Path.DirectorySeparatorChar))) ?? rootPath;
var absoluteMapPath = Path.GetFullPath(Path.Combine(sourceDirectory, mapPath));
if (!File.Exists(absoluteMapPath))
{
return;
}
try
{
var info = new FileInfo(absoluteMapPath);
if (info.Length > MaxSourceMapBytes)
{
return;
}
using var stream = File.OpenRead(absoluteMapPath);
using var document = JsonDocument.Parse(stream);
if (!document.RootElement.TryGetProperty("sources", out var sourcesElement) || sourcesElement.ValueKind != JsonValueKind.Array)
{
return;
}
var contents = document.RootElement.TryGetProperty("sourcesContent", out var sourcesContent) && sourcesContent.ValueKind == JsonValueKind.Array
? sourcesContent
: default;
for (var i = 0; i < sourcesElement.GetArrayLength(); i++)
{
var sourceEntry = sourcesElement[i];
if (sourceEntry.ValueKind != JsonValueKind.String)
{
continue;
}
var mapSourcePath = sourceEntry.GetString();
if (string.IsNullOrWhiteSpace(mapSourcePath))
{
continue;
}
var combinedSourcePath = NormalizeRelative(rootPath, Path.GetFullPath(Path.Combine(sourceDirectory, mapSourcePath)));
var sourceContent = contents.ValueKind == JsonValueKind.Array && contents.GetArrayLength() > i && contents[i].ValueKind == JsonValueKind.String
? contents[i].GetString()
: null;
if (string.IsNullOrEmpty(sourceContent))
{
continue;
}
AnalyzeInternal(rootPath, combinedSourcePath, sourceContent!, allowSourceMap: false, edges);
}
}
catch (IOException)
{
// ignore unreadable source maps
}
catch (JsonException)
{
// ignore malformed source maps
}
}
private static string NormalizeRelative(string rootPath, string absolutePath)
{
var normalizedRoot = Path.GetFullPath(rootPath);
var normalizedPath = Path.GetFullPath(absolutePath);
if (!normalizedPath.StartsWith(normalizedRoot, StringComparison.Ordinal))
{
return normalizedPath.Replace('\\', '/');
}
var relative = Path.GetRelativePath(normalizedRoot, normalizedPath);
return relative.Replace('\\', '/');
}
private static string? ExtractSourceMapPath(string content)
{
if (string.IsNullOrWhiteSpace(content))
{
return null;
}
var match = Regex.Match(content, "sourceMappingURL=([^\n\\s]+)");
return match.Success ? match.Groups[1].Value.Trim() : null;
}
private static bool IsRequire(CallExpression call)
{
return call.Callee is Identifier id && string.Equals(id.Name, "require", StringComparison.Ordinal)
&& call.Arguments.Count == 1 && call.Arguments[0] is Literal { Value: string };
&& call.Arguments.Count == 1;
}
private static string CombineConfidence(string left, string right)
{
static int Score(string value) => value switch
{
"high" => 3,
"low" => 1,
_ => 2
};
var combined = Math.Min(Score(left), Score(right));
return combined switch
{
3 => "high",
2 => "medium",
_ => "low"
};
}
}

View File

@@ -0,0 +1,149 @@
using System.Collections.Immutable;
using System.IO;
using System.Linq;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
internal sealed record NodeProjectInput(
IReadOnlyList<string> SourceRoots,
IReadOnlyList<string> NodeModuleRoots,
IReadOnlyList<string> Tarballs,
IReadOnlyList<string> YarnCacheRoots,
bool YarnPnpPresent);
/// <summary>
/// Normalizes scanner inputs for Node.js projects, layering workspace roots, container layers,
/// pnpm stores, Yarn Plug'n'Play caches, and tarball sources into a deterministic view.
/// </summary>
internal static class NodeInputNormalizer
{
private static readonly string[] LayerRootCandidates = { "layers", ".layers", "layer" };
public static NodeProjectInput Normalize(LanguageAnalyzerContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var sourceRoots = DiscoverSourceRoots(context.RootPath);
var nodeModuleRoots = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var tarballs = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var yarnCacheRoots = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var root in sourceRoots)
{
cancellationToken.ThrowIfCancellationRequested();
var nodeModules = Path.Combine(root, "node_modules");
if (Directory.Exists(nodeModules))
{
nodeModuleRoots.Add(Path.GetFullPath(nodeModules));
}
foreach (var candidate in EnumerateTarballs(root))
{
tarballs.Add(candidate);
}
var yarnCache = Path.Combine(root, ".yarn", "cache");
if (Directory.Exists(yarnCache))
{
yarnCacheRoots.Add(Path.GetFullPath(yarnCache));
}
}
var workspaceIndex = NodeWorkspaceIndex.Create(context.RootPath);
foreach (var workspace in workspaceIndex.GetMembers())
{
var absolute = Path.GetFullPath(Path.Combine(context.RootPath, workspace.Replace('/', Path.DirectorySeparatorChar)));
var workspaceNodeModules = Path.Combine(absolute, "node_modules");
if (Directory.Exists(workspaceNodeModules))
{
nodeModuleRoots.Add(workspaceNodeModules);
}
}
var yarnPnpPresent = sourceRoots.Any(static root => HasYarnPnpMarkers(root));
return new NodeProjectInput(
SourceRoots: sourceRoots.OrderBy(static p => p, StringComparer.Ordinal).ToArray(),
NodeModuleRoots: nodeModuleRoots.OrderBy(static p => p, StringComparer.Ordinal).ToArray(),
Tarballs: tarballs.OrderBy(static p => p, StringComparer.Ordinal).ToArray(),
YarnCacheRoots: yarnCacheRoots.OrderBy(static p => p, StringComparer.Ordinal).ToArray(),
YarnPnpPresent: yarnPnpPresent);
}
private static IReadOnlyList<string> DiscoverSourceRoots(string rootPath)
{
var roots = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{
Path.GetFullPath(rootPath)
};
foreach (var candidateRoot in LayerRootCandidates)
{
var path = Path.Combine(rootPath, candidateRoot);
if (!Directory.Exists(path))
{
continue;
}
foreach (var child in SafeEnumerateDirectories(path))
{
roots.Add(Path.GetFullPath(child));
}
}
foreach (var child in SafeEnumerateDirectories(rootPath))
{
var name = Path.GetFileName(child);
if (name.StartsWith("layer", StringComparison.OrdinalIgnoreCase))
{
roots.Add(Path.GetFullPath(child));
}
}
return roots.ToImmutableArray();
}
private static IEnumerable<string> SafeEnumerateDirectories(string path)
{
try
{
return Directory.EnumerateDirectories(path);
}
catch
{
return Array.Empty<string>();
}
}
private static IEnumerable<string> EnumerateTarballs(string rootPath)
{
var options = new EnumerationOptions
{
RecurseSubdirectories = true,
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.Device
};
try
{
return Directory
.EnumerateFiles(rootPath, "*.tgz", options)
.Select(Path.GetFullPath)
.ToArray();
}
catch
{
return Array.Empty<string>();
}
}
private static bool HasYarnPnpMarkers(string rootPath)
{
var pnpCjs = Path.Combine(rootPath, ".pnp.cjs");
var pnpData = Path.Combine(rootPath, ".pnp.data.json");
var yarnCache = Path.Combine(rootPath, ".yarn", "cache");
return File.Exists(pnpCjs) || File.Exists(pnpData) || Directory.Exists(yarnCache);
}
}

View File

@@ -1,4 +1,6 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
@@ -84,9 +86,11 @@ internal sealed class NodePackage
private readonly List<NodeEntrypoint> _entrypoints = new();
private readonly List<NodeImportEdge> _imports = new();
private readonly List<NodeImportResolution> _resolvedImports = new();
public IReadOnlyList<NodeEntrypoint> Entrypoints => _entrypoints;
public IReadOnlyList<NodeImportEdge> Imports => _imports;
public IReadOnlyList<NodeImportResolution> ResolvedImports => _resolvedImports;
public string RelativePathNormalized => string.IsNullOrEmpty(RelativePath) ? string.Empty : RelativePath.Replace(Path.DirectorySeparatorChar, '/');
@@ -150,11 +154,30 @@ internal sealed class NodePackage
foreach (var importEdge in _imports.OrderBy(static e => e.ComparisonKey, StringComparer.Ordinal))
{
var value = string.IsNullOrWhiteSpace(importEdge.Evidence)
? $"{importEdge.TargetSpecifier} (conf:{importEdge.Confidence})"
: $"{importEdge.TargetSpecifier} (conf:{importEdge.Confidence};{importEdge.Evidence})";
evidence.Add(new LanguageComponentEvidence(
LanguageEvidenceKind.File,
"node.import",
importEdge.SourceFile,
importEdge.TargetSpecifier,
value,
null));
}
foreach (var resolution in _resolvedImports.OrderBy(static r => r.SourceFile, StringComparer.Ordinal)
.ThenBy(static r => r.Specifier, StringComparer.Ordinal))
{
var locator = string.IsNullOrWhiteSpace(PackageJsonLocator)
? "package.json"
: PackageJsonLocator;
evidence.Add(new LanguageComponentEvidence(
LanguageEvidenceKind.Metadata,
"node.resolve",
locator,
$"{resolution.SourceFile}:{resolution.Specifier}->{resolution.ResolvedPath} ({resolution.ResolutionType};{resolution.Confidence})",
null));
}
@@ -322,7 +345,7 @@ internal sealed class NodePackage
return;
}
var edge = new NodeImportEdge(sourceFile.Replace(Path.DirectorySeparatorChar, '/'), targetSpecifier.Trim(), kind.Trim(), evidence);
var edge = new NodeImportEdge(sourceFile.Replace(Path.DirectorySeparatorChar, '/'), targetSpecifier.Trim(), kind.Trim(), evidence, "high");
if (_imports.Any(e => string.Equals(e.ComparisonKey, edge.ComparisonKey, StringComparison.Ordinal)))
{
return;
@@ -331,6 +354,41 @@ internal sealed class NodePackage
_imports.Add(edge);
}
public void AddImport(NodeImportEdge edge)
{
if (edge is null)
{
return;
}
if (_imports.Any(e => string.Equals(e.ComparisonKey, edge.ComparisonKey, StringComparison.Ordinal)))
{
return;
}
_imports.Add(edge);
}
public void SetResolvedImports(IEnumerable<NodeImportResolution> resolutions)
{
_resolvedImports.Clear();
if (resolutions is null)
{
return;
}
foreach (var resolution in resolutions)
{
if (resolution is null)
{
continue;
}
_resolvedImports.Add(resolution);
}
}
private static IEnumerable<string> ParseConditionSet(string conditionSet)
{
if (string.IsNullOrWhiteSpace(conditionSet))

View File

@@ -13,15 +13,15 @@ internal static class NodePackageCollector
"__pycache__"
};
public static IReadOnlyList<NodePackage> CollectPackages(LanguageAnalyzerContext context, NodeLockData lockData, CancellationToken cancellationToken)
public static IReadOnlyList<NodePackage> CollectPackages(LanguageAnalyzerContext context, NodeLockData lockData, NodeProjectInput projectInput, CancellationToken cancellationToken)
{
var packages = new List<NodePackage>();
var visited = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var pendingNodeModuleRoots = new List<string>();
var nodeModuleRoots = new HashSet<string>(projectInput.NodeModuleRoots, StringComparer.OrdinalIgnoreCase);
var rootPackageJson = Path.Combine(context.RootPath, "package.json");
var workspaceIndex = NodeWorkspaceIndex.Create(context.RootPath);
var yarnPnpPresent = HasYarnPnp(context.RootPath);
var yarnPnpPresent = projectInput.YarnPnpPresent;
if (File.Exists(rootPackageJson))
{
@@ -46,25 +46,28 @@ internal static class NodePackageCollector
var workspaceNodeModules = Path.Combine(workspaceAbsolute, "node_modules");
if (Directory.Exists(workspaceNodeModules))
{
pendingNodeModuleRoots.Add(workspaceNodeModules);
nodeModuleRoots.Add(workspaceNodeModules);
}
}
var nodeModules = Path.Combine(context.RootPath, "node_modules");
TraverseDirectory(context, nodeModules, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
foreach (var pendingRoot in pendingNodeModuleRoots.OrderBy(static path => path, StringComparer.Ordinal))
foreach (var nodeModules in nodeModuleRoots.OrderBy(static path => path, StringComparer.Ordinal))
{
TraverseDirectory(context, pendingRoot, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
TraverseDirectory(context, nodeModules, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
}
TraverseTarballs(context, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken);
TraverseYarnPnpCache(context, packages, visited, yarnPnpPresent, cancellationToken);
TraverseTarballs(context, projectInput.Tarballs, packages, visited, yarnPnpPresent, cancellationToken);
TraverseYarnPnpCache(context, projectInput.YarnCacheRoots, packages, visited, yarnPnpPresent, cancellationToken);
AppendDeclaredPackages(packages, lockData);
AttachImports(context, packages, cancellationToken);
var resolutions = NodeResolver.Resolve(context, projectInput, packages, cancellationToken);
foreach (var (package, resolvedImports) in resolutions)
{
package.SetResolvedImports(resolvedImports);
}
return packages;
}
@@ -97,10 +100,11 @@ internal static class NodePackageCollector
continue;
}
var imports = NodeImportWalker.AnalyzeImports(context.GetRelativePath(file).Replace(Path.DirectorySeparatorChar, '/'), content);
var relativeSource = context.GetRelativePath(file).Replace(Path.DirectorySeparatorChar, '/');
var imports = NodeImportWalker.AnalyzeImports(context.RootPath, relativeSource, content);
foreach (var edge in imports)
{
package.AddImport(edge.SourceFile, edge.TargetSpecifier, edge.Kind, edge.Evidence);
package.AddImport(edge);
}
}
}
@@ -248,21 +252,13 @@ internal static class NodePackageCollector
private static void TraverseTarballs(
LanguageAnalyzerContext context,
NodeLockData lockData,
NodeWorkspaceIndex workspaceIndex,
IEnumerable<string> tarballPaths,
List<NodePackage> packages,
HashSet<string> visited,
bool yarnPnpPresent,
CancellationToken cancellationToken)
{
var enumerationOptions = new EnumerationOptions
{
RecurseSubdirectories = true,
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.Device
};
foreach (var tgzPath in Directory.EnumerateFiles(context.RootPath, "*.tgz", enumerationOptions))
foreach (var tgzPath in tarballPaths)
{
cancellationToken.ThrowIfCancellationRequested();
TryProcessTarball(context, tgzPath, packages, visited, yarnPnpPresent, cancellationToken);
@@ -353,6 +349,7 @@ internal static class NodePackageCollector
private static void TraverseYarnPnpCache(
LanguageAnalyzerContext context,
IEnumerable<string> cacheRoots,
List<NodePackage> packages,
HashSet<string> visited,
bool yarnPnpPresent,
@@ -363,12 +360,6 @@ internal static class NodePackageCollector
return;
}
var cacheDirectory = Path.Combine(context.RootPath, ".yarn", "cache");
if (!Directory.Exists(cacheDirectory))
{
return;
}
var enumerationOptions = new EnumerationOptions
{
RecurseSubdirectories = true,
@@ -376,10 +367,18 @@ internal static class NodePackageCollector
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.Device
};
foreach (var zipPath in Directory.EnumerateFiles(cacheDirectory, "*.zip", enumerationOptions))
foreach (var cacheDirectory in cacheRoots)
{
cancellationToken.ThrowIfCancellationRequested();
TryProcessZipball(context, zipPath, packages, visited, yarnPnpPresent, cancellationToken);
if (!Directory.Exists(cacheDirectory))
{
continue;
}
foreach (var zipPath in Directory.EnumerateFiles(cacheDirectory, "*.zip", enumerationOptions))
{
cancellationToken.ThrowIfCancellationRequested();
TryProcessZipball(context, zipPath, packages, visited, yarnPnpPresent, cancellationToken);
}
}
}
@@ -730,22 +729,6 @@ internal static class NodePackageCollector
return IgnoredDirectories.Any(ignored => string.Equals(name, ignored, StringComparison.OrdinalIgnoreCase));
}
private static bool HasYarnPnp(string rootPath)
{
if (string.IsNullOrWhiteSpace(rootPath))
{
return false;
}
var pnpCjs = Path.Combine(rootPath, ".pnp.cjs");
var pnpData = Path.Combine(rootPath, ".pnp.data.json");
var yarnCache = Path.Combine(rootPath, ".yarn", "cache");
return File.Exists(pnpCjs)
|| File.Exists(pnpData)
|| Directory.Exists(yarnCache);
}
private static IReadOnlyList<string> ExtractWorkspaceTargets(string relativeDirectory, JsonElement root, NodeWorkspaceIndex workspaceIndex)
{
var dependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "dependencies"));
@@ -903,6 +886,24 @@ internal static class NodePackageCollector
}
}
if (root.TryGetProperty("imports", out var importsElement))
{
foreach (var importEntry in FlattenExports(importsElement, prefix: "imports"))
{
AddEntrypoint(importEntry.Path, importEntry.Conditions, binName: null, mainField: null, moduleField: null);
}
}
if (root.TryGetProperty("worker", out var workerElement) && workerElement.ValueKind == JsonValueKind.String)
{
AddEntrypoint(workerElement.GetString(), "worker", binName: null, mainField: null, moduleField: null);
}
if (HasElectronDependency(root) && root.TryGetProperty("main", out var electronMain) && electronMain.ValueKind == JsonValueKind.String)
{
AddEntrypoint(electronMain.GetString(), "electron", binName: null, mainField: electronMain.GetString(), moduleField: null);
}
DetectShebangEntrypoints(context, package, relativeDirectory);
}
@@ -919,7 +920,7 @@ internal static class NodePackageCollector
yield break;
case JsonValueKind.Object:
foreach (var property in element.EnumerateObject())
foreach (var property in element.EnumerateObject().OrderBy(static p => p.Name, StringComparer.Ordinal))
{
var nextPrefix = string.IsNullOrWhiteSpace(prefix) ? property.Name : $"{prefix},{property.Name}";
foreach (var nested in FlattenExports(property.Value, nextPrefix))
@@ -934,6 +935,30 @@ internal static class NodePackageCollector
}
}
private static bool HasElectronDependency(JsonElement root)
{
static bool ContainsElectron(JsonElement? element)
{
if (element is null || element.Value.ValueKind != JsonValueKind.Object)
{
return false;
}
foreach (var property in element.Value.EnumerateObject())
{
if (string.Equals(property.Name, "electron", StringComparison.OrdinalIgnoreCase))
{
return true;
}
}
return false;
}
return ContainsElectron(TryGetProperty(root, "dependencies"))
|| ContainsElectron(TryGetProperty(root, "devDependencies"));
}
private static void DetectShebangEntrypoints(LanguageAnalyzerContext context, NodePackage package, string relativeDirectory)
{
var baseDirectory = string.IsNullOrWhiteSpace(relativeDirectory)

View File

@@ -0,0 +1,532 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
/// <summary>
/// Resolves Node.js import specifiers using a deterministic, offline-friendly subset of the
/// Node resolution algorithm (CJS + ESM). Handles core modules, relative/absolute paths,
/// self-references, exports/imports maps, and classic node_modules search with extension
/// priorities.
/// </summary>
internal static class NodeResolver
{
private static readonly string[] ExtensionPriority = { string.Empty, ".js", ".mjs", ".cjs", ".ts", ".tsx", ".json", ".node" };
private static readonly HashSet<string> CoreModules = new(
new[]
{
"fs","path","http","https","net","tls","dns","os","stream","buffer","crypto","url",
"util","events","child_process","cluster","readline","zlib","assert","querystring","perf_hooks",
"inspector","module","timers","tty","vm","worker_threads","diagnostics_channel","process"
}, StringComparer.Ordinal);
public static IReadOnlyList<(NodePackage Package, IReadOnlyList<NodeImportResolution> ResolvedImports)> Resolve(
LanguageAnalyzerContext context,
NodeProjectInput projectInput,
IReadOnlyCollection<NodePackage> packages,
CancellationToken cancellationToken)
{
var results = new List<(NodePackage, IReadOnlyList<NodeImportResolution>)>();
var packageJsonCache = new Dictionary<string, JsonDocument>(StringComparer.OrdinalIgnoreCase);
foreach (var package in packages)
{
cancellationToken.ThrowIfCancellationRequested();
var resolved = ResolvePackage(context, projectInput, package, packageJsonCache, cancellationToken);
if (resolved.Count > 0)
{
results.Add((package, resolved));
}
}
return results;
}
private static IReadOnlyList<NodeImportResolution> ResolvePackage(
LanguageAnalyzerContext context,
NodeProjectInput projectInput,
NodePackage package,
Dictionary<string, JsonDocument> packageJsonCache,
CancellationToken cancellationToken)
{
if (package.Imports.Count == 0)
{
return Array.Empty<NodeImportResolution>();
}
var resolved = new List<NodeImportResolution>();
var packageRoot = ResolvePackageRoot(context.RootPath, package.RelativePathNormalized);
var searchOrder = BuildNodeModulesSearchOrder(packageRoot, projectInput.NodeModuleRoots);
var packageJson = LoadPackageJson(packageRoot, packageJsonCache);
foreach (var edge in package.Imports)
{
cancellationToken.ThrowIfCancellationRequested();
var resolution = ResolveImport(context, packageRoot, searchOrder, packageJson, edge);
resolved.Add(resolution);
}
return resolved
.OrderBy(static r => r.SourceFile, StringComparer.Ordinal)
.ThenBy(static r => r.Specifier, StringComparer.Ordinal)
.ToArray();
}
private static NodeImportResolution ResolveImport(
LanguageAnalyzerContext context,
string packageRoot,
IReadOnlyList<string> nodeModuleSearchOrder,
JsonDocument? packageJson,
NodeImportEdge edge)
{
if (IsCoreModule(edge.TargetSpecifier))
{
return new NodeImportResolution(edge.SourceFile, edge.TargetSpecifier, edge.TargetSpecifier, "core", edge.Confidence);
}
if (edge.TargetSpecifier.StartsWith('#'))
{
var mapped = ResolveImportsMap(packageRoot, packageJson, edge.TargetSpecifier);
if (!string.IsNullOrWhiteSpace(mapped))
{
var resolvedPath = ResolvePath(context, packageRoot, mapped!, nodeModuleSearchOrder, packageJson, out var resolutionType);
return resolvedPath is null
? CreateUnresolved(edge)
: new NodeImportResolution(edge.SourceFile, edge.TargetSpecifier, resolvedPath, resolutionType, CombineConfidence(edge.Confidence, "medium"));
}
}
if (IsRelative(edge.TargetSpecifier) || edge.TargetSpecifier.StartsWith('/'))
{
var resolvedPath = ResolvePath(context, packageRoot, edge.TargetSpecifier, nodeModuleSearchOrder, packageJson, out var resolutionType);
return resolvedPath is null
? CreateUnresolved(edge)
: new NodeImportResolution(edge.SourceFile, edge.TargetSpecifier, resolvedPath, resolutionType, edge.Confidence);
}
if (IsSelfReference(packageJson, edge.TargetSpecifier))
{
var selfTarget = TrimSelfPrefix(packageJson, edge.TargetSpecifier);
var resolvedPath = ResolvePath(context, packageRoot, selfTarget, nodeModuleSearchOrder, packageJson, out var resolutionType);
return resolvedPath is null
? CreateUnresolved(edge)
: new NodeImportResolution(edge.SourceFile, edge.TargetSpecifier, resolvedPath, resolutionType, CombineConfidence(edge.Confidence, "medium"));
}
var bareResolved = ResolveBareSpecifier(context, packageRoot, nodeModuleSearchOrder, edge.TargetSpecifier, out var bareType);
if (bareResolved is not null)
{
return new NodeImportResolution(edge.SourceFile, edge.TargetSpecifier, bareResolved, bareType, edge.Confidence);
}
return CreateUnresolved(edge);
}
private static string? ResolveBareSpecifier(
LanguageAnalyzerContext context,
string packageRoot,
IReadOnlyList<string> nodeModuleSearchOrder,
string specifier,
out string resolutionType)
{
resolutionType = "unresolved";
foreach (var nodeModules in nodeModuleSearchOrder)
{
var candidate = Path.Combine(nodeModules, specifier);
if (Directory.Exists(candidate))
{
var resolved = ResolveDirectoryEntrypoint(context, candidate, out resolutionType);
if (resolved is not null)
{
return resolved;
}
}
var fileResolved = ResolveFile(candidate);
if (fileResolved is not null)
{
resolutionType = "file";
return ToRelative(context, fileResolved);
}
}
return null;
}
private static string? ResolvePath(
LanguageAnalyzerContext context,
string packageRoot,
string target,
IReadOnlyList<string> nodeModuleSearchOrder,
JsonDocument? packageJson,
out string resolutionType)
{
resolutionType = "unresolved";
var normalized = target.Replace('/', Path.DirectorySeparatorChar);
var basePath = target.StartsWith('/')
? Path.Combine(context.RootPath, normalized.TrimStart(Path.DirectorySeparatorChar))
: Path.GetFullPath(Path.Combine(packageRoot, normalized));
var fileResolved = ResolveFile(basePath);
if (fileResolved is not null)
{
resolutionType = "file";
return ToRelative(context, fileResolved);
}
if (Directory.Exists(basePath))
{
var directoryResolved = ResolveDirectoryEntrypoint(context, basePath, out resolutionType, packageJson);
if (directoryResolved is not null)
{
return directoryResolved;
}
}
// Last resort: treat as bare specifier against node_modules search order
var bare = ResolveBareSpecifier(context, packageRoot, nodeModuleSearchOrder, target, out resolutionType);
if (bare is not null)
{
return bare;
}
return null;
}
private static string? ResolveDirectoryEntrypoint(
LanguageAnalyzerContext context,
string directory,
out string resolutionType,
JsonDocument? packageJson = null)
{
resolutionType = "directory";
var packageJsonPath = Path.Combine(directory, "package.json");
JsonDocument? localPackage = null;
if (File.Exists(packageJsonPath))
{
localPackage = LoadPackageJson(packageJsonPath, new());
var exportsTarget = ResolveExports(localPackage);
if (!string.IsNullOrWhiteSpace(exportsTarget))
{
var resolved = ResolveFileOrDirectory(context, directory, exportsTarget!);
if (resolved is not null)
{
resolutionType = "exports";
return resolved;
}
}
if (localPackage is not null && TryGetString(localPackage, "main", out var mainValue))
{
var resolved = ResolveFileOrDirectory(context, directory, mainValue!);
if (resolved is not null)
{
resolutionType = "main";
return resolved;
}
}
if (localPackage is not null && TryGetString(localPackage, "module", out var moduleValue))
{
var resolved = ResolveFileOrDirectory(context, directory, moduleValue!);
if (resolved is not null)
{
resolutionType = "module";
return resolved;
}
}
}
foreach (var ext in ExtensionPriority)
{
var indexCandidate = Path.Combine(directory, "index" + ext);
if (File.Exists(indexCandidate))
{
return ToRelative(context, indexCandidate);
}
}
return null;
}
private static string? ResolveFileOrDirectory(LanguageAnalyzerContext context, string directory, string target)
{
var normalized = target.Replace('/', Path.DirectorySeparatorChar);
var candidate = Path.GetFullPath(Path.Combine(directory, normalized));
var file = ResolveFile(candidate);
if (file is not null)
{
return ToRelative(context, file);
}
if (Directory.Exists(candidate))
{
var index = ResolveDirectoryEntrypoint(context, candidate, out _);
if (index is not null)
{
return index;
}
}
return null;
}
private static string? ResolveExports(JsonDocument? packageJson)
{
if (packageJson is null)
{
return null;
}
if (!packageJson.RootElement.TryGetProperty("exports", out var exportsElement))
{
return null;
}
if (exportsElement.ValueKind == JsonValueKind.String)
{
return exportsElement.GetString();
}
if (exportsElement.ValueKind != JsonValueKind.Object)
{
return null;
}
// Prefer default > import > require > node > browser > worker for determinism
var preferred = new[] { "default", "import", "require", "node", "browser", "worker" };
foreach (var key in preferred)
{
if (exportsElement.TryGetProperty(key, out var prop) && prop.ValueKind == JsonValueKind.String)
{
return prop.GetString();
}
}
return null;
}
private static string? ResolveImportsMap(string packageRoot, JsonDocument? packageJson, string specifier)
{
if (packageJson is null)
{
return null;
}
if (!packageJson.RootElement.TryGetProperty("imports", out var importsElement) || importsElement.ValueKind != JsonValueKind.Object)
{
return null;
}
if (!importsElement.TryGetProperty(specifier, out var mapped))
{
return null;
}
if (mapped.ValueKind == JsonValueKind.String)
{
return mapped.GetString();
}
if (mapped.ValueKind == JsonValueKind.Object)
{
foreach (var property in mapped.EnumerateObject().OrderBy(static p => p.Name, StringComparer.Ordinal))
{
if (property.Value.ValueKind == JsonValueKind.String)
{
return property.Value.GetString();
}
}
}
return null;
}
private static string? ResolveFile(string candidate)
{
if (File.Exists(candidate))
{
return candidate;
}
foreach (var ext in ExtensionPriority.Skip(1))
{
var withExt = candidate + ext;
if (File.Exists(withExt))
{
return withExt;
}
}
return null;
}
private static JsonDocument? LoadPackageJson(string packageRoot, Dictionary<string, JsonDocument> cache)
{
if (string.IsNullOrWhiteSpace(packageRoot))
{
return null;
}
var packageJsonPath = Directory.Exists(packageRoot)
? Path.Combine(packageRoot, "package.json")
: packageRoot;
if (!File.Exists(packageJsonPath))
{
return null;
}
if (cache.TryGetValue(packageJsonPath, out var cached))
{
return cached;
}
try
{
var document = JsonDocument.Parse(File.ReadAllText(packageJsonPath));
cache[packageJsonPath] = document;
return document;
}
catch
{
return null;
}
}
private static IReadOnlyList<string> BuildNodeModulesSearchOrder(string packageRoot, IReadOnlyList<string> globalNodeModuleRoots)
{
var searchRoots = new List<string>();
var current = packageRoot;
while (!string.IsNullOrWhiteSpace(current))
{
var nodeModules = Path.Combine(current, "node_modules");
if (Directory.Exists(nodeModules))
{
searchRoots.Add(Path.GetFullPath(nodeModules));
}
var parent = Directory.GetParent(current);
if (parent is null || string.Equals(parent.FullName, current, StringComparison.Ordinal))
{
break;
}
current = parent.FullName;
}
foreach (var root in globalNodeModuleRoots)
{
if (Directory.Exists(root))
{
searchRoots.Add(Path.GetFullPath(root));
}
}
return searchRoots
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static r => r, StringComparer.Ordinal)
.ToArray();
}
private static string ResolvePackageRoot(string rootPath, string relativeDirectory)
{
if (string.IsNullOrWhiteSpace(relativeDirectory))
{
return rootPath;
}
return Path.GetFullPath(Path.Combine(rootPath, relativeDirectory.Replace('/', Path.DirectorySeparatorChar)));
}
private static bool IsCoreModule(string specifier) => CoreModules.Contains(specifier);
private static bool IsRelative(string specifier) => specifier.StartsWith("./") || specifier.StartsWith("../");
private static bool IsSelfReference(JsonDocument? packageJson, string specifier)
{
if (packageJson is null)
{
return false;
}
if (!packageJson.RootElement.TryGetProperty("name", out var nameElement))
{
return false;
}
var name = nameElement.GetString();
if (string.IsNullOrWhiteSpace(name))
{
return false;
}
return string.Equals(specifier, name, StringComparison.Ordinal)
|| specifier.StartsWith(name + "/", StringComparison.Ordinal);
}
private static string TrimSelfPrefix(JsonDocument? packageJson, string specifier)
{
if (packageJson is null || !packageJson.RootElement.TryGetProperty("name", out var nameElement))
{
return specifier;
}
var name = nameElement.GetString();
if (string.IsNullOrWhiteSpace(name))
{
return specifier;
}
return specifier.Length > name.Length + 1
? specifier[(name.Length + 1)..]
: "";
}
private static bool TryGetString(JsonDocument document, string property, out string? value)
{
if (document.RootElement.TryGetProperty(property, out var element) && element.ValueKind == JsonValueKind.String)
{
value = element.GetString();
return true;
}
value = null;
return false;
}
private static string ToRelative(LanguageAnalyzerContext context, string absolutePath)
{
var relative = context.GetRelativePath(absolutePath);
return string.IsNullOrWhiteSpace(relative) ? absolutePath.Replace('\\', '/') : relative.Replace('\\', '/');
}
private static NodeImportResolution CreateUnresolved(NodeImportEdge edge)
=> new(edge.SourceFile, edge.TargetSpecifier, "unresolved", "unresolved", "low");
private static string CombineConfidence(string left, string right)
{
static int Score(string value) => value switch
{
"high" => 3,
"low" => 1,
_ => 2
};
var combined = Math.Min(Score(left), Score(right));
return combined switch
{
3 => "high",
2 => "medium",
_ => "low"
};
}
}

View File

@@ -0,0 +1,679 @@
using System.Buffers.Binary;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using StellaOps.Scanner.Analyzers.Lang;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal.Phase22;
internal static class NodePhase22Analyzer
{
private const long MaxMapBytes = 50L * 1024 * 1024; // 50 MB
public static NodePhase22Observation Analyze(LanguageAnalyzerContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var records = new List<NodePhase22Record>();
AnalyzeBundles(context, records, cancellationToken);
AnalyzeNativeAndWasm(context, records, cancellationToken);
AnalyzeCapabilities(context, records, cancellationToken);
var ordered = records
.Where(static r => r.Confidence is null || r.Confidence >= 0.4)
.Where(static r => r.ResolverTrace.Count > 0)
.Distinct(NodePhase22Record.Comparer)
.OrderBy(static r => r.Type, StringComparer.Ordinal)
.ThenBy(static r => r.Path ?? r.From ?? string.Empty, StringComparer.Ordinal)
.ThenBy(static r => r.To ?? string.Empty, StringComparer.Ordinal)
.ToArray();
return new NodePhase22Observation(ordered);
}
private static void AnalyzeBundles(LanguageAnalyzerContext context, List<NodePhase22Record> records, CancellationToken cancellationToken)
{
foreach (var scriptPath in EnumerateFiles(context.RootPath, [".js", ".mjs", ".cjs"], cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
string content;
try
{
content = File.ReadAllText(scriptPath);
}
catch (IOException)
{
continue;
}
var mapRef = TryFindSourceMapReference(content);
if (mapRef is null)
{
continue;
}
var bundlePath = NormalizePath(context, scriptPath);
var resolverTrace = new List<string> { $"bundle:{bundlePath}" };
var map = TryLoadSourceMap(context, scriptPath, mapRef, cancellationToken);
if (map is not null)
{
resolverTrace.Add(map.MapTrace);
foreach (var source in map.Sources)
{
var record = new NodePhase22Record(
Type: "component",
ComponentType: "pkg",
EdgeType: null,
Path: source,
From: null,
To: null,
Format: map.Format,
FromBundle: true,
Reason: "source-map",
Confidence: 0.87,
ResolverTrace: new[] { resolverTrace[0], resolverTrace[1], $"source:{source}" },
Exports: null,
Arch: null,
Platform: null);
records.Add(record);
}
}
// Always emit entrypoint to signal bundle presence (even when map rejected)
var entryTrace = new List<string>(resolverTrace);
if (map is null && mapRef.Length > 0)
{
entryTrace.Add($"map:{mapRef}");
}
records.Add(new NodePhase22Record(
Type: "entrypoint",
ComponentType: null,
EdgeType: null,
Path: bundlePath,
From: null,
To: null,
Format: "esm",
FromBundle: null,
Reason: "bundle-entrypoint",
Confidence: map is null ? 0.51 : 0.88,
ResolverTrace: entryTrace,
Exports: null,
Arch: null,
Platform: null));
}
}
private static SourceMapResult? TryLoadSourceMap(LanguageAnalyzerContext context, string scriptPath, string mapReference, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(mapReference))
{
return null;
}
try
{
if (mapReference.StartsWith("data:application/json;base64,", StringComparison.OrdinalIgnoreCase))
{
var base64 = mapReference["data:application/json;base64,".Length..];
var bytes = Convert.FromBase64String(base64);
using var inlineDoc = JsonDocument.Parse(bytes);
return BuildSourceMapResult(inlineDoc.RootElement, "map:inline", isInline: true);
}
var mapPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(scriptPath)!, mapReference));
if (!File.Exists(mapPath))
{
return null;
}
var info = new FileInfo(mapPath);
if (info.Length > MaxMapBytes)
{
return null;
}
using var stream = File.OpenRead(mapPath);
using var mapDoc = JsonDocument.Parse(stream);
return BuildSourceMapResult(mapDoc.RootElement, $"map:{NormalizePath(context, mapPath)}", isInline: false);
}
catch (FormatException)
{
return null;
}
catch (JsonException)
{
return null;
}
catch (IOException)
{
return null;
}
}
private static SourceMapResult? BuildSourceMapResult(JsonElement root, string mapTrace, bool isInline)
{
if (!root.TryGetProperty("sources", out var sourcesElement) || sourcesElement.ValueKind != JsonValueKind.Array)
{
return null;
}
if (!root.TryGetProperty("sourcesContent", out var sourcesContent) || sourcesContent.ValueKind != JsonValueKind.Array)
{
return null;
}
var sources = new List<string>();
foreach (var item in sourcesElement.EnumerateArray())
{
if (item.ValueKind != JsonValueKind.String)
{
continue;
}
var normalized = NormalizeSourceMapPath(item.GetString());
if (!string.IsNullOrWhiteSpace(normalized))
{
sources.Add(normalized);
}
}
if (sources.Count == 0)
{
return null;
}
return new SourceMapResult(sources, mapTrace, Format: "esm", isInline);
}
private static void AnalyzeNativeAndWasm(LanguageAnalyzerContext context, List<NodePhase22Record> records, CancellationToken cancellationToken)
{
var nativeFiles = EnumerateFiles(context.RootPath, [".node"], cancellationToken).ToArray();
var wasmFiles = EnumerateFiles(context.RootPath, [".wasm"], cancellationToken).ToArray();
foreach (var nativePath in nativeFiles)
{
cancellationToken.ThrowIfCancellationRequested();
var (arch, platform) = TryDetectNativeMetadata(nativePath);
var normalized = NormalizePath(context, nativePath);
records.Add(new NodePhase22Record(
Type: "component",
ComponentType: "native",
EdgeType: null,
Path: normalized,
From: null,
To: null,
Format: null,
FromBundle: null,
Reason: "native-addon-file",
Confidence: 0.82,
ResolverTrace: new[] { $"file:{normalized}" },
Exports: null,
Arch: arch,
Platform: platform));
}
foreach (var wasmPath in wasmFiles)
{
cancellationToken.ThrowIfCancellationRequested();
var normalized = NormalizePath(context, wasmPath);
records.Add(new NodePhase22Record(
Type: "component",
ComponentType: "wasm",
EdgeType: null,
Path: normalized,
From: null,
To: null,
Format: null,
FromBundle: null,
Reason: "wasm-file",
Confidence: 0.80,
ResolverTrace: new[] { $"file:{normalized}" },
Exports: null,
Arch: null,
Platform: null));
}
}
private static void AnalyzeCapabilities(LanguageAnalyzerContext context, List<NodePhase22Record> records, CancellationToken cancellationToken)
{
foreach (var scriptPath in EnumerateFiles(context.RootPath, [".js", ".mjs", ".cjs", ".ts", ".tsx"], cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
string content;
try
{
content = File.ReadAllText(scriptPath);
}
catch (IOException)
{
continue;
}
var normalizedSource = NormalizePath(context, scriptPath);
foreach (var edge in ExtractNativeEdges(content, normalizedSource, scriptPath, context))
{
records.Add(edge);
}
foreach (var edge in ExtractWasmEdges(content, normalizedSource, scriptPath, context))
{
records.Add(edge);
}
foreach (var capability in ExtractCapabilityEdges(content, normalizedSource))
{
records.Add(capability);
}
}
}
private static IEnumerable<NodePhase22Record> ExtractNativeEdges(
string content,
string sourcePath,
string sourceAbsolute,
LanguageAnalyzerContext context)
{
var pattern = "process.dlopen";
var index = content.IndexOf(pattern, StringComparison.Ordinal);
while (index >= 0)
{
var argument = TryExtractLiteral(content, index + pattern.Length);
if (!string.IsNullOrWhiteSpace(argument))
{
var target = NormalizeTarget(context, sourceAbsolute, argument!);
yield return new NodePhase22Record(
Type: "edge",
ComponentType: null,
EdgeType: "native-addon",
Path: null,
From: sourcePath,
To: target,
Format: null,
FromBundle: null,
Reason: "native-dlopen-string",
Confidence: 0.76,
ResolverTrace: new[] { $"source:{sourcePath}", $"call:process.dlopen('{argument}')" },
Exports: null,
Arch: null,
Platform: null);
}
index = content.IndexOf(pattern, index + pattern.Length, StringComparison.Ordinal);
}
}
private static IEnumerable<NodePhase22Record> ExtractWasmEdges(
string content,
string sourcePath,
string sourceAbsolute,
LanguageAnalyzerContext context)
{
var pattern = "WebAssembly.instantiate";
var index = content.IndexOf(pattern, StringComparison.Ordinal);
while (index >= 0)
{
var argument = TryExtractLiteral(content, index + pattern.Length);
if (!string.IsNullOrWhiteSpace(argument))
{
var target = NormalizeTarget(context, sourceAbsolute, argument!);
yield return new NodePhase22Record(
Type: "edge",
ComponentType: null,
EdgeType: "wasm",
Path: null,
From: sourcePath,
To: target,
Format: null,
FromBundle: null,
Reason: "wasm-import",
Confidence: 0.74,
ResolverTrace: new[] { $"source:{sourcePath}", $"call:WebAssembly.instantiate('{argument}')" },
Exports: null,
Arch: null,
Platform: null);
}
index = content.IndexOf(pattern, index + pattern.Length, StringComparison.Ordinal);
}
}
private static IEnumerable<NodePhase22Record> ExtractCapabilityEdges(string content, string sourcePath)
{
if (content.Contains("child_process", StringComparison.Ordinal))
{
yield return Capability(sourcePath, "child_process.execFile", "capability-child-process");
}
if (content.Contains("worker_threads", StringComparison.Ordinal))
{
yield return Capability(sourcePath, "worker_threads", "capability-worker");
}
if (content.Contains("process.binding", StringComparison.Ordinal))
{
yield return Capability(sourcePath, "process.binding", "capability-binding");
}
if (content.Contains("vm.", StringComparison.Ordinal))
{
yield return Capability(sourcePath, "vm", "capability-vm");
}
if (content.Contains("fs.promises", StringComparison.Ordinal))
{
yield return Capability(sourcePath, "fs.promises", "capability-fs-promises");
}
}
private static NodePhase22Record Capability(string sourcePath, string target, string reason)
{
return new NodePhase22Record(
Type: "edge",
ComponentType: null,
EdgeType: "capability",
Path: null,
From: sourcePath,
To: target,
Format: null,
FromBundle: null,
Reason: reason,
Confidence: 0.70,
ResolverTrace: new[] { $"source:{sourcePath}", $"call:{target}" },
Exports: null,
Arch: null,
Platform: null);
}
private static string NormalizeTarget(LanguageAnalyzerContext context, string sourceAbsolute, string argument)
{
if (string.IsNullOrWhiteSpace(argument))
{
return argument;
}
if (!argument.StartsWith('.'))
{
return argument.Replace('\\', '/');
}
var sourceDirectory = Path.GetDirectoryName(sourceAbsolute);
var baseDir = string.IsNullOrWhiteSpace(sourceDirectory) ? context.RootPath : sourceDirectory!;
var combined = Path.GetFullPath(Path.Combine(baseDir, argument));
var relative = context.GetRelativePath(combined).Replace('\\', '/');
return relative.StartsWith('/') ? relative : "/" + relative;
}
private static (string? Arch, string? Platform) TryDetectNativeMetadata(string path)
{
try
{
Span<byte> header = stackalloc byte[64];
using var stream = File.OpenRead(path);
var read = stream.Read(header);
if (read >= 5 && header[0] == 0x7F && header[1] == (byte)'E' && header[2] == (byte)'L' && header[3] == (byte)'F')
{
var elfClass = header[4];
var eMachine = read > 0x13 ? BinaryPrimitives.ReadUInt16LittleEndian(header[0x12..]) : (ushort)0;
return (elfClass == 2 ? "x86_64" : "x86", "linux" + (eMachine != 0 ? string.Empty : string.Empty));
}
if (read >= 2 && header[0] == 0x4D && header[1] == 0x5A)
{
return ("x86_64", "windows");
}
if (read >= 4 && ((header[0] == 0xFE && header[1] == 0xED) || (header[0] == 0xCE && header[1] == 0xFA)))
{
return ("x86_64", "macos");
}
}
catch (IOException)
{
// ignore unreadable native file
}
return (null, null);
}
private static string NormalizePath(LanguageAnalyzerContext context, string absolutePath)
{
var relative = context.GetRelativePath(absolutePath).Replace('\\', '/');
return relative.StartsWith('/') ? relative : "/" + relative;
}
private static string NormalizeSourceMapPath(string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return string.Empty;
}
var normalized = value.Replace("webpack://", string.Empty, StringComparison.OrdinalIgnoreCase)
.Replace('\\', '/');
while (normalized.Contains("../", StringComparison.Ordinal))
{
normalized = normalized.Replace("../", string.Empty, StringComparison.Ordinal);
}
if (!normalized.StartsWith('/'))
{
normalized = "/" + normalized.TrimStart('/');
}
return normalized;
}
private static IEnumerable<string> EnumerateFiles(string root, string[] extensions, CancellationToken cancellationToken)
{
var options = new EnumerationOptions
{
RecurseSubdirectories = true,
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.Device
};
foreach (var ext in extensions)
{
foreach (var file in Directory.EnumerateFiles(root, "*" + ext, options))
{
cancellationToken.ThrowIfCancellationRequested();
yield return file;
}
}
}
private static string? TryExtractLiteral(string content, int startIndex)
{
var quoteStart = content.IndexOf('"', startIndex);
var altQuoteStart = content.IndexOf('\'', startIndex);
if (quoteStart < 0 || (altQuoteStart >= 0 && altQuoteStart < quoteStart))
{
quoteStart = altQuoteStart;
}
if (quoteStart < 0 || quoteStart >= content.Length - 1)
{
return null;
}
var quoteChar = content[quoteStart];
var end = content.IndexOf(quoteChar, quoteStart + 1);
if (end <= quoteStart)
{
return null;
}
return content.Substring(quoteStart + 1, end - quoteStart - 1).Trim();
}
private static string? TryFindSourceMapReference(string content)
{
const string marker = "sourceMappingURL=";
var index = content.LastIndexOf(marker, StringComparison.OrdinalIgnoreCase);
if (index < 0)
{
return null;
}
var start = index + marker.Length;
var end = content.IndexOf('\n', start);
if (end < 0)
{
end = content.Length;
}
var value = content[start..end].Trim().TrimEnd('*', '/');
return value;
}
}
internal sealed record NodePhase22Record(
string Type,
string? ComponentType,
string? EdgeType,
string? Path,
string? From,
string? To,
string? Format,
bool? FromBundle,
string? Reason,
double? Confidence,
IReadOnlyList<string> ResolverTrace,
IReadOnlyList<string>? Exports,
string? Arch,
string? Platform)
{
public static IEqualityComparer<NodePhase22Record> Comparer { get; } = new NodePhase22RecordComparer();
private sealed class NodePhase22RecordComparer : IEqualityComparer<NodePhase22Record>
{
public bool Equals(NodePhase22Record? x, NodePhase22Record? y)
{
if (ReferenceEquals(x, y))
{
return true;
}
if (x is null || y is null)
{
return false;
}
return string.Equals(x.Type, y.Type, StringComparison.Ordinal)
&& string.Equals(x.ComponentType, y.ComponentType, StringComparison.Ordinal)
&& string.Equals(x.EdgeType, y.EdgeType, StringComparison.Ordinal)
&& string.Equals(x.Path ?? x.From, y.Path ?? y.From, StringComparison.Ordinal)
&& string.Equals(x.To, y.To, StringComparison.Ordinal)
&& string.Equals(x.Reason, y.Reason, StringComparison.Ordinal)
&& x.ResolverTrace.SequenceEqual(y.ResolverTrace);
}
public int GetHashCode(NodePhase22Record obj)
{
var hash = new HashCode();
hash.Add(obj.Type, StringComparer.Ordinal);
hash.Add(obj.ComponentType, StringComparer.Ordinal);
hash.Add(obj.EdgeType, StringComparer.Ordinal);
hash.Add(obj.Path ?? obj.From, StringComparer.Ordinal);
hash.Add(obj.To, StringComparer.Ordinal);
hash.Add(obj.Reason, StringComparer.Ordinal);
foreach (var step in obj.ResolverTrace)
{
hash.Add(step, StringComparer.Ordinal);
}
return hash.ToHashCode();
}
}
}
internal sealed class NodePhase22Observation
{
public NodePhase22Observation(IReadOnlyList<NodePhase22Record> records)
{
Records = records ?? Array.Empty<NodePhase22Record>();
}
public IReadOnlyList<NodePhase22Record> Records { get; }
public bool HasRecords => Records.Count > 0;
public int EntrypointCount => Records.Count(r => string.Equals(r.Type, "entrypoint", StringComparison.Ordinal));
public int ComponentCount => Records.Count(r => string.Equals(r.Type, "component", StringComparison.Ordinal));
public int EdgeCount => Records.Count(r => string.Equals(r.Type, "edge", StringComparison.Ordinal));
public int NativeCount => Records.Count(r => string.Equals(r.ComponentType, "native", StringComparison.Ordinal));
public int WasmCount => Records.Count(r => string.Equals(r.ComponentType, "wasm", StringComparison.Ordinal));
public string ToNdjson()
{
if (!HasRecords)
{
return string.Empty;
}
var options = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
};
var builder = new StringBuilder();
foreach (var record in Records)
{
var line = JsonSerializer.Serialize(record, options);
builder.AppendLine(line);
}
return builder.ToString().TrimEnd();
}
public string ComputeSha256()
{
if (!HasRecords)
{
return string.Empty;
}
var json = ToNdjson();
var bytes = Encoding.UTF8.GetBytes(json);
return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
}
public IEnumerable<KeyValuePair<string, string?>> BuildMetadata()
{
yield return new KeyValuePair<string, string?>("node.observation.entrypoints", EntrypointCount.ToString(CultureInfo.InvariantCulture));
yield return new KeyValuePair<string, string?>("node.observation.components", ComponentCount.ToString(CultureInfo.InvariantCulture));
yield return new KeyValuePair<string, string?>("node.observation.edges", EdgeCount.ToString(CultureInfo.InvariantCulture));
if (NativeCount > 0)
{
yield return new KeyValuePair<string, string?>("node.observation.native", NativeCount.ToString(CultureInfo.InvariantCulture));
}
if (WasmCount > 0)
{
yield return new KeyValuePair<string, string?>("node.observation.wasm", WasmCount.ToString(CultureInfo.InvariantCulture));
}
}
}
internal sealed record SourceMapResult(IReadOnlyList<string> Sources, string MapTrace, string Format, bool IsInline);

View File

@@ -0,0 +1,175 @@
using System.Globalization;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal.Phase22;
internal static class NodePhase22Exporter
{
public static IReadOnlyList<LanguageComponentRecord> ToComponentRecords(NodePhase22Observation observation)
{
if (observation is null || !observation.HasRecords)
{
return Array.Empty<LanguageComponentRecord>();
}
var records = new List<LanguageComponentRecord>();
// Observation envelope
var ndjson = observation.ToNdjson();
var sha256 = observation.ComputeSha256();
records.Add(LanguageComponentRecord.FromExplicitKey(
analyzerId: "node",
componentKey: "observation::node-phase22",
purl: null,
name: "Node Observation (Phase 22)",
version: null,
type: "node-observation",
metadata: observation.BuildMetadata(),
evidence: new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.Derived,
"node.observation",
"phase22.ndjson",
ndjson,
string.IsNullOrWhiteSpace(sha256) ? null : sha256)
},
usedByEntrypoint: false));
foreach (var record in observation.Records)
{
if (string.Equals(record.Type, "component", StringComparison.Ordinal))
{
records.Add(ConvertComponent(record));
}
else if (string.Equals(record.Type, "edge", StringComparison.Ordinal))
{
records.Add(ConvertEdge(record));
}
else if (string.Equals(record.Type, "entrypoint", StringComparison.Ordinal))
{
records.Add(ConvertEntrypoint(record));
}
}
return records;
}
private static LanguageComponentRecord ConvertComponent(NodePhase22Record record)
{
var typeTag = record.ComponentType switch
{
"native" => "node:native",
"wasm" => "node:wasm",
_ => "node:bundle"
};
var metadata = new List<KeyValuePair<string, string?>>();
if (!string.IsNullOrWhiteSpace(record.Reason)) metadata.Add(new("reason", record.Reason));
if (!string.IsNullOrWhiteSpace(record.Format)) metadata.Add(new("format", record.Format));
if (record.Confidence is double conf) metadata.Add(new("confidence", conf.ToString("0.00", CultureInfo.InvariantCulture)));
if (record.FromBundle is bool fromBundle) metadata.Add(new("fromBundle", fromBundle ? "true" : "false"));
if (record.ResolverTrace.Count > 0) metadata.Add(new("trace", string.Join("|", record.ResolverTrace)));
if (!string.IsNullOrWhiteSpace(record.Arch)) metadata.Add(new("arch", record.Arch));
if (!string.IsNullOrWhiteSpace(record.Platform)) metadata.Add(new("platform", record.Platform));
var evidence = record.ResolverTrace.Count == 0
? null
: new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.Metadata,
"node.trace",
record.ResolverTrace[0],
record.ResolverTrace.Count > 1 ? string.Join("|", record.ResolverTrace) : null,
null)
};
var name = record.Path is null ? "" : Path.GetFileName(record.Path.Trim('/'));
return LanguageComponentRecord.FromExplicitKey(
analyzerId: "node-phase22",
componentKey: record.Path ?? Guid.NewGuid().ToString("N"),
purl: null,
name: string.IsNullOrWhiteSpace(name) ? "node-component" : name,
version: null,
type: typeTag,
metadata: metadata,
evidence: evidence,
usedByEntrypoint: false);
}
private static LanguageComponentRecord ConvertEdge(NodePhase22Record record)
{
var metadata = new List<KeyValuePair<string, string?>>
{
new("from", record.From ?? string.Empty),
new("to", record.To ?? string.Empty)
};
if (!string.IsNullOrWhiteSpace(record.Reason)) metadata.Add(new("reason", record.Reason));
if (record.Confidence is double conf) metadata.Add(new("confidence", conf.ToString("0.00", CultureInfo.InvariantCulture)));
if (record.ResolverTrace.Count > 0) metadata.Add(new("trace", string.Join("|", record.ResolverTrace)));
var evidence = record.ResolverTrace.Count == 0
? null
: new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.Derived,
"node.edge",
record.ResolverTrace[0],
record.ResolverTrace.Count > 1 ? string.Join("|", record.ResolverTrace) : null,
null)
};
var key = string.Concat("edge:", record.From ?? string.Empty, "->", record.To ?? string.Empty, "#", record.EdgeType ?? "edge");
return LanguageComponentRecord.FromExplicitKey(
analyzerId: "node-phase22",
componentKey: key,
purl: null,
name: record.EdgeType ?? "edge",
version: null,
type: "node:edge",
metadata: metadata,
evidence: evidence,
usedByEntrypoint: false);
}
private static LanguageComponentRecord ConvertEntrypoint(NodePhase22Record record)
{
var metadata = new List<KeyValuePair<string, string?>>
{
new("entrypoint", record.Path ?? string.Empty)
};
if (!string.IsNullOrWhiteSpace(record.Format)) metadata.Add(new("format", record.Format));
if (record.Confidence is double conf) metadata.Add(new("confidence", conf.ToString("0.00", CultureInfo.InvariantCulture)));
if (record.ResolverTrace.Count > 0) metadata.Add(new("trace", string.Join("|", record.ResolverTrace)));
var evidence = record.ResolverTrace.Count == 0
? null
: new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.File,
"node.entrypoint",
record.Path ?? "entrypoint",
record.ResolverTrace.Count > 0 ? record.ResolverTrace[0] : null,
null)
};
var name = record.Path is null ? "entrypoint" : Path.GetFileName(record.Path.Trim('/'));
return LanguageComponentRecord.FromExplicitKey(
analyzerId: "node-phase22",
componentKey: record.Path ?? Guid.NewGuid().ToString("N"),
purl: null,
name: string.IsNullOrWhiteSpace(name) ? "entrypoint" : name,
version: null,
type: "node:entrypoint",
metadata: metadata,
evidence: evidence,
usedByEntrypoint: true);
}
}

View File

@@ -0,0 +1,170 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text.Json;
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal;
/// <summary>
/// Ingests optional runtime evidence produced by Node loader hooks (ESM/CJS).
/// Input format: NDJSON records under <root>/node-runtime-evidence.ndjson or path from SCANNER_NODE_RUNTIME_EVIDENCE.
/// Each line is a JSON object with fields:
/// type: "edge" | "component"
/// from, to: optional strings
/// reason: string (e.g., runtime-import, runtime-require)
/// loaderId: optional string to be SHA-256 hashed
/// path: optional component path
/// </summary>
internal static class RuntimeEvidenceLoader
{
private const string DefaultFileName = "node-runtime-evidence.ndjson";
private const string EnvKey = "SCANNER_NODE_RUNTIME_EVIDENCE";
public static IReadOnlyList<LanguageComponentRecord> Load(LanguageAnalyzerContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var path = Environment.GetEnvironmentVariable(EnvKey);
if (string.IsNullOrWhiteSpace(path))
{
path = Path.Combine(context.RootPath, DefaultFileName);
}
if (!File.Exists(path))
{
return Array.Empty<LanguageComponentRecord>();
}
var records = new List<LanguageComponentRecord>();
using var stream = File.OpenRead(path);
using var reader = new StreamReader(stream);
string? line;
while ((line = reader.ReadLine()) is not null)
{
cancellationToken.ThrowIfCancellationRequested();
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
try
{
using var doc = JsonDocument.Parse(line);
var root = doc.RootElement;
var kind = root.TryGetProperty("type", out var typeProp) ? typeProp.GetString() : null;
if (string.IsNullOrWhiteSpace(kind))
{
continue;
}
var loaderId = TryHash(root, "loaderId");
var reason = root.TryGetProperty("reason", out var reasonProp) ? reasonProp.GetString() : null;
var from = root.TryGetProperty("from", out var fromProp) ? ScrubPath(context, fromProp.GetString()) : null;
var to = root.TryGetProperty("to", out var toProp) ? ScrubPath(context, toProp.GetString()) : null;
var componentPath = root.TryGetProperty("path", out var pathProp) ? ScrubPath(context, pathProp.GetString()) : null;
var metadata = new List<KeyValuePair<string, string?>>();
if (!string.IsNullOrWhiteSpace(reason)) metadata.Add(new("reason", reason));
if (!string.IsNullOrWhiteSpace(loaderId)) metadata.Add(new("loaderId.sha256", loaderId));
if (string.Equals(kind, "edge", StringComparison.Ordinal))
{
if (!string.IsNullOrWhiteSpace(from)) metadata.Add(new("from", from));
if (!string.IsNullOrWhiteSpace(to)) metadata.Add(new("to", to));
var evidence = BuildDerivedEvidence(reason, from, to);
records.Add(LanguageComponentRecord.FromExplicitKey(
analyzerId: "node-runtime",
componentKey: string.Concat("runtime-edge:", from ?? "", "->", to ?? ""),
purl: null,
name: "runtime-edge",
version: null,
type: "node:runtime-edge",
metadata: metadata,
evidence: evidence,
usedByEntrypoint: false));
}
else if (string.Equals(kind, "component", StringComparison.Ordinal))
{
var name = string.IsNullOrWhiteSpace(componentPath) ? "runtime-component" : Path.GetFileName(componentPath);
metadata.Add(new("path", componentPath));
records.Add(LanguageComponentRecord.FromExplicitKey(
analyzerId: "node-runtime",
componentKey: componentPath ?? Guid.NewGuid().ToString("N"),
purl: null,
name: name,
version: null,
type: "node:runtime-component",
metadata: metadata,
evidence: BuildDerivedEvidence(reason, from, to),
usedByEntrypoint: false));
}
}
catch (JsonException)
{
continue;
}
}
return records
.OrderBy(static r => r.ComponentKey, StringComparer.Ordinal)
.ToArray();
}
private static IReadOnlyList<LanguageComponentEvidence>? BuildDerivedEvidence(string? reason, string? from, string? to)
{
var locatorParts = new[] { reason, from, to }
.Where(static v => !string.IsNullOrWhiteSpace(v));
var locator = string.Join("|", locatorParts);
if (string.IsNullOrWhiteSpace(locator))
{
return null;
}
return new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.Derived,
"node.runtime",
locator,
null,
null)
};
}
private static string? TryHash(JsonElement root, string property)
{
if (!root.TryGetProperty(property, out var prop) || prop.ValueKind != JsonValueKind.String)
{
return null;
}
var value = prop.GetString();
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
var bytes = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(value));
return Convert.ToHexString(bytes).ToLowerInvariant();
}
private static string? ScrubPath(LanguageAnalyzerContext context, string? value)
{
if (string.IsNullOrWhiteSpace(value))
{
return value;
}
var trimmed = value.Trim();
if (!Path.IsPathRooted(trimmed))
{
return trimmed.Replace('\\', '/');
}
var relative = context.GetRelativePath(trimmed).Replace('\\', '/');
return string.IsNullOrWhiteSpace(relative) ? trimmed.Replace('\\', '/') : relative;
}
}

View File

@@ -15,14 +15,15 @@ public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer
ArgumentNullException.ThrowIfNull(writer);
var lockData = await NodeLockData.LoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
var packages = NodePackageCollector.CollectPackages(context, lockData, cancellationToken);
var projectInput = NodeInputNormalizer.Normalize(context, cancellationToken);
var packages = NodePackageCollector.CollectPackages(context, lockData, projectInput, cancellationToken);
foreach (var package in packages.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal))
{
cancellationToken.ThrowIfCancellationRequested();
var metadata = package.CreateMetadata();
var evidence = package.CreateEvidence();
cancellationToken.ThrowIfCancellationRequested();
var metadata = package.CreateMetadata();
var evidence = package.CreateEvidence();
writer.AddFromPurl(
analyzerId: Id,
@@ -35,11 +36,59 @@ public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer
usedByEntrypoint: package.IsUsedByEntrypoint);
}
// Optional Phase 22 prep path: ingest precomputed bundle/native/WASM AOC records from NDJSON fixture
var phase22Records = await NodePhase22SampleLoader.TryLoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
if (phase22Records.Count > 0)
var observation = NodePhase22Analyzer.Analyze(context, cancellationToken);
if (observation.HasRecords)
{
var ndjson = observation.ToNdjson();
var sha256 = observation.ComputeSha256();
var evidence = new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.Derived,
"node.observation",
"phase22.ndjson",
ndjson,
sha256)
};
writer.AddFromExplicitKey(
analyzerId: Id,
componentKey: "observation::node-phase22",
purl: null,
name: "Node Observation (Phase 22)",
version: null,
type: "node-observation",
metadata: observation.BuildMetadata(),
evidence: evidence);
}
else
{
// Fallback to NDJSON fixture when running against prep-only environments.
var phase22Records = await NodePhase22SampleLoader.TryLoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
if (phase22Records.Count > 0)
{
writer.AddRange(phase22Records);
}
var observation = NodePhase22Analyzer.Analyze(context, cancellationToken);
if (observation.HasRecords)
{
var observationRecords = NodePhase22Exporter.ToComponentRecords(observation);
writer.AddRange(observationRecords);
}
var runtimeRecords = RuntimeEvidenceLoader.Load(context, cancellationToken);
if (runtimeRecords.Count > 0)
{
writer.AddRange(runtimeRecords);
}
var envWarnings = NodeEnvironmentScanner.Scan(context, projectInput.SourceRoots, cancellationToken);
if (envWarnings.Count > 0)
{
writer.AddRange(envWarnings);
}
}
}
}

View File

@@ -18,6 +18,15 @@
<PackageReference Include="Esprima" Version="3.0.5" />
</ItemGroup>
<ItemGroup>
<None Update="runtime-hooks\runtime-require-hook.js">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="runtime-hooks\runtime-esm-loader.mjs">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Scanner.Analyzers.Lang\StellaOps.Scanner.Analyzers.Lang.csproj" />
</ItemGroup>

View File

@@ -0,0 +1,16 @@
# Node Analyzer Tasks (Sprint 132)
| Task ID | Status | Notes | Updated (UTC) |
| --- | --- | --- | --- |
| SCANNER-ANALYZERS-NODE-22-001 | DONE | VFS/input normalizer covers dirs/tgz/container layers/pnpm/Yarn PnP; Node version detection wired. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-002 | DONE | Entrypoint discovery extended (exports/imports/workers/electron/shebang) with normalized condition sets. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-003 | DONE | Import walker flags dynamic patterns with confidence and de-bundles source maps. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-004 | DONE | Resolver engine added (core modules, exports/imports maps, extension priority, self references). | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-005 | DONE | Yarn PnP and pnpm virtual store adapters supported via VFS; tests updated. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-006 | DONE | Bundle/source-map correlation emits component/entrypoint records with resolver traces. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-007 | DONE | Native addon/WASM/capability edges produced with normalized targets. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-008 | DONE | Phase22 observation export (entrypoints/components/edges) added to analyzer output. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-009 | DONE | Fixture suite refreshed (npm/pnpm/PnP/bundle/electron/worker) with golden outputs. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-010 | DONE | Runtime evidence hooks (ESM loader/CJS require) with path scrubbing and hashed loader IDs; ingestion to runtime-* records. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-011 | DONE | Packaged plug-in manifest (0.1.0) with runtime hooks; CLI/offline docs refreshed. | 2025-12-01 |
| SCANNER-ANALYZERS-NODE-22-012 | DONE | Container filesystem adapter (layer roots) + NODE_OPTIONS/env warnings emitted. | 2025-12-01 |

View File

@@ -0,0 +1,61 @@
// Runtime ESM loader for StellaOps Scanner runtime evidence
// Usage: node --experimental-loader=./runtime-esm-loader.mjs app.mjs
import fs from 'fs';
import path from 'path';
import crypto from 'crypto';
import { fileURLToPath, pathToFileURL } from 'url';
const outPath = process.env.SCANNER_NODE_RUNTIME_OUT || path.join(process.cwd(), 'node-runtime-evidence.ndjson');
const root = process.env.SCANNER_NODE_ROOT || process.cwd();
const loaderId = hashLoaderId(import.meta.url);
function hashLoaderId(value) {
return crypto.createHash('sha256').update(value || '').digest('hex');
}
function scrub(p) {
if (!p) return p;
try {
const absolute = p.startsWith('file:') ? fileURLToPath(p) : p;
const rel = path.relative(root, absolute);
return rel.startsWith('..') ? p : rel.split(path.sep).join('/');
} catch {
return p;
}
}
function emit(record) {
try {
fs.appendFileSync(outPath, JSON.stringify(record) + '\n');
} catch {
// best-effort: ignore write failures
}
}
export async function resolve(specifier, context, next) {
const parent = context.parentURL ? scrub(context.parentURL) : undefined;
const target = scrub(specifier);
emit({
type: 'edge',
from: parent,
to: target,
reason: 'runtime-import',
loaderId
});
return next(specifier, context, next);
}
export async function load(url, context, next) {
const pathOrUrl = scrub(url);
emit({
type: 'component',
path: pathOrUrl,
reason: 'runtime-load',
loaderId
});
return next(url, context, next);
}

View File

@@ -0,0 +1,49 @@
// Runtime require hook for StellaOps Scanner runtime evidence
// Usage: node -r ./runtime-require-hook.js app.js
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const outPath = process.env.SCANNER_NODE_RUNTIME_OUT || path.join(process.cwd(), 'node-runtime-evidence.ndjson');
const root = process.env.SCANNER_NODE_ROOT || process.cwd();
function hashLoaderId(value) {
return crypto.createHash('sha256').update(value || '').digest('hex');
}
function scrub(p) {
if (!p) return p;
try {
const rel = path.relative(root, p);
return rel.startsWith('..') ? p : rel.split(path.sep).join('/');
} catch {
return p;
}
}
function emit(record) {
try {
fs.appendFileSync(outPath, JSON.stringify(record) + '\n');
} catch {
// best-effort: ignore write failures
}
}
const originalLoad = module.constructor._load;
module.constructor._load = function (request, parent, isMain) {
const from = parent && parent.filename ? scrub(parent.filename) : undefined;
const to = scrub(request);
const loaderId = hashLoaderId(__filename);
emit({
type: 'edge',
from,
to,
reason: 'runtime-require',
loaderId,
isMain: !!isMain
});
return originalLoad.apply(this, arguments);
};

View File

@@ -23,10 +23,11 @@ public static class ComponentGraphBuilder
{
ArgumentNullException.ThrowIfNull(fragments);
var orderedLayers = fragments
.Where(static fragment => !string.IsNullOrWhiteSpace(fragment.LayerDigest))
.Select(NormalizeFragment)
.ToImmutableArray();
var orderedLayers = fragments
.Where(static fragment => !string.IsNullOrWhiteSpace(fragment.LayerDigest))
.Select(NormalizeFragment)
.OrderBy(static fragment => fragment.LayerDigest, StringComparer.Ordinal)
.ToImmutableArray();
var accumulators = new Dictionary<string, ComponentAccumulator>(StringComparer.Ordinal);

View File

@@ -35,14 +35,14 @@ public sealed class CycloneDxComposer
var graph = ComponentGraphBuilder.Build(request.LayerFragments);
var generatedAt = ScannerTimestamps.Normalize(request.GeneratedAt);
var inventoryArtifact = BuildArtifact(
request,
graph,
SbomView.Inventory,
graph.Components,
generatedAt,
InventoryMediaTypeJson,
InventoryMediaTypeProtobuf);
var inventoryArtifact = BuildArtifact(
request,
graph,
SbomView.Inventory,
graph.Components,
generatedAt,
InventoryMediaTypeJson,
InventoryMediaTypeProtobuf);
var usageComponents = graph.Components
.Where(static component => component.Usage.UsedByEntrypoint)
@@ -51,14 +51,14 @@ public sealed class CycloneDxComposer
CycloneDxArtifact? usageArtifact = null;
if (!usageComponents.IsEmpty)
{
usageArtifact = BuildArtifact(
request,
graph,
SbomView.Usage,
usageComponents,
generatedAt,
UsageMediaTypeJson,
UsageMediaTypeProtobuf);
usageArtifact = BuildArtifact(
request,
graph,
SbomView.Usage,
usageComponents,
generatedAt,
UsageMediaTypeJson,
UsageMediaTypeProtobuf);
}
return new SbomCompositionResult
@@ -69,37 +69,47 @@ public sealed class CycloneDxComposer
};
}
private CycloneDxArtifact BuildArtifact(
SbomCompositionRequest request,
ComponentGraph graph,
SbomView view,
ImmutableArray<AggregatedComponent> components,
DateTimeOffset generatedAt,
string jsonMediaType,
string protobufMediaType)
{
private CycloneDxArtifact BuildArtifact(
SbomCompositionRequest request,
ComponentGraph graph,
SbomView view,
ImmutableArray<AggregatedComponent> components,
DateTimeOffset generatedAt,
string jsonMediaType,
string protobufMediaType)
{
var bom = BuildBom(request, graph, view, components, generatedAt);
var json = JsonSerializer.Serialize(bom);
var jsonBytes = Encoding.UTF8.GetBytes(json);
var protobufBytes = ProtoSerializer.Serialize(bom);
var json = JsonSerializer.Serialize(bom);
var jsonBytes = Encoding.UTF8.GetBytes(json);
var protobufBytes = ProtoSerializer.Serialize(bom);
var jsonHash = ComputeSha256(jsonBytes);
var protobufHash = ComputeSha256(protobufBytes);
var merkleRoot = request.AdditionalProperties is not null
&& request.AdditionalProperties.TryGetValue("stellaops:merkle.root", out var root)
? root
: null;
request.AdditionalProperties?.TryGetValue("stellaops:composition.manifest", out var compositionUri);
var jsonHash = ComputeSha256(jsonBytes);
var protobufHash = ComputeSha256(protobufBytes);
return new CycloneDxArtifact
{
View = view,
SerialNumber = bom.SerialNumber ?? string.Empty,
GeneratedAt = generatedAt,
Components = components,
JsonBytes = jsonBytes,
JsonSha256 = jsonHash,
JsonMediaType = jsonMediaType,
ProtobufBytes = protobufBytes,
ProtobufSha256 = protobufHash,
ProtobufMediaType = protobufMediaType,
};
}
return new CycloneDxArtifact
{
View = view,
SerialNumber = bom.SerialNumber ?? string.Empty,
GeneratedAt = generatedAt,
Components = components,
JsonBytes = jsonBytes,
JsonSha256 = jsonHash,
ContentHash = jsonHash,
MerkleRoot = merkleRoot,
CompositionUri = compositionUri,
JsonMediaType = jsonMediaType,
ProtobufBytes = protobufBytes,
ProtobufSha256 = protobufHash,
ProtobufMediaType = protobufMediaType,
};
}
private Bom BuildBom(
SbomCompositionRequest request,

View File

@@ -4,25 +4,40 @@ using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Emit.Composition;
public sealed record CycloneDxArtifact
{
public required SbomView View { get; init; }
public required string SerialNumber { get; init; }
public sealed record CycloneDxArtifact
{
public required SbomView View { get; init; }
public required string SerialNumber { get; init; }
public required DateTimeOffset GeneratedAt { get; init; }
public required ImmutableArray<AggregatedComponent> Components { get; init; }
public required byte[] JsonBytes { get; init; }
public required string JsonSha256 { get; init; }
public required string JsonMediaType { get; init; }
public required byte[] ProtobufBytes { get; init; }
public required string ProtobufSha256 { get; init; }
public required byte[] JsonBytes { get; init; }
public required string JsonSha256 { get; init; }
/// <summary>
/// Canonical content hash (sha256, hex) of the CycloneDX JSON payload.
/// </summary>
public required string ContentHash { get; init; }
/// <summary>
/// Merkle root over fragments (hex). Present when composition metadata is provided.
/// </summary>
public string? MerkleRoot { get; init; }
/// <summary>
/// CAS URI of the composition recipe (_composition.json) if emitted.
/// </summary>
public string? CompositionUri { get; init; }
public required string JsonMediaType { get; init; }
public required byte[] ProtobufBytes { get; init; }
public required string ProtobufSha256 { get; init; }
public required string ProtobufMediaType { get; init; }
}

View File

@@ -96,7 +96,7 @@ public sealed class EntryTraceRuntimeReconciler
terminalBuilder[index] = terminalBuilder[index] with { Confidence = confidence.Score };
}
diagnostics.Add(BuildDiagnostic(confidence, plan.TerminalPath));
diagnostics.Add(BuildDiagnostic(confidence, plan.TerminalPath, procGraph, match?.Process));
}
// Update any terminals that were not tied to plans.
@@ -242,7 +242,7 @@ public sealed class EntryTraceRuntimeReconciler
return new ConfidenceResult(60d, ConfidenceLevel.Low, runtimePath);
}
private static EntryTraceDiagnostic BuildDiagnostic(ConfidenceResult result, string predictedPath)
private EntryTraceDiagnostic BuildDiagnostic(ConfidenceResult result, string predictedPath, ProcGraph procGraph, ProcProcess? process)
{
var runtimePath = string.IsNullOrWhiteSpace(result.RuntimePath) ? "<unknown>" : result.RuntimePath;
var severity = result.Level == ConfidenceLevel.High
@@ -251,10 +251,18 @@ public sealed class EntryTraceRuntimeReconciler
var reason = result.Level == ConfidenceLevel.High
? EntryTraceUnknownReason.RuntimeMatch
: EntryTraceUnknownReason.RuntimeMismatch;
var chain = process is null ? null : BuildProcessChain(procGraph, process.Value);
var message = result.Level == ConfidenceLevel.High
? $"Runtime process '{runtimePath}' matches EntryTrace prediction '{predictedPath}'."
: $"Runtime process '{runtimePath}' diverges from EntryTrace prediction '{predictedPath}'.";
if (!string.IsNullOrWhiteSpace(chain))
{
message += $" Runtime chain: {chain}.";
}
return new EntryTraceDiagnostic(
severity,
reason,
@@ -269,6 +277,50 @@ public sealed class EntryTraceRuntimeReconciler
return command.Length > 0 && WrapperNames.Contains(command);
}
private static string? BuildProcessChain(ProcGraph graph, ProcProcess process)
{
var chain = new List<string>();
var current = process;
while (true)
{
var display = string.IsNullOrWhiteSpace(current.ExecutablePath)
? current.CommandName
: current.ExecutablePath;
if (string.IsNullOrWhiteSpace(display))
{
display = current.CommandName;
}
chain.Add(display);
if (current.ParentPid == current.Pid || current.ParentPid == 0 || !graph.Processes.TryGetValue(current.ParentPid, out var parent))
{
break;
}
current = parent;
}
chain.Reverse();
// Collapse adjacent wrappers to a single token for readability.
var collapsed = new List<string>(chain.Count);
foreach (var segment in chain)
{
var name = Path.GetFileName(segment);
var isWrapper = WrapperNames.Contains(name);
if (isWrapper && collapsed.Count > 0 && WrapperNames.Contains(Path.GetFileName(collapsed[^1])))
{
continue; // skip duplicate adjacent wrapper entries
}
collapsed.Add(segment);
}
return collapsed.Count == 0 ? null : string.Join(" -> ", collapsed);
}
private static string GetCommandName(ProcProcess process)
{
if (!string.IsNullOrWhiteSpace(process.CommandName))

View File

@@ -3,3 +3,6 @@
| Task ID | Status | Date | Summary |
| --- | --- | --- | --- |
| SCANNER-ENG-0008 | DONE | 2025-11-16 | Documented quarterly EntryTrace heuristic cadence and workflow; attached to Sprint 0138 Execution Log. |
| SCANNER-ENTRYTRACE-18-504 | DONE | 2025-12-01 | EntryTrace NDJSON emission and streaming (entry/node/edge/target/warning/capability) wired via Worker → WebService/CLI. |
| SCANNER-ENTRYTRACE-18-505 | DONE | 2025-12-01 | Runtime ProcGraph reconciliation adjusts plan/terminal confidence and diagnostics for matches/mismatches. |
| SCANNER-ENTRYTRACE-18-506 | DONE | 2025-12-01 | EntryTrace graph/NDJSON exposed via WebService `/scans/{id}/entrytrace` and CLI rendering. |

View File

@@ -0,0 +1,32 @@
using StellaOps.Scanner.Analyzers.Lang;
namespace StellaOps.Scanner.Analyzers.Lang.Node.SmokeTests;
internal static class LanguageAnalyzerSmokeHarness
{
public static async Task AssertDeterministicAsync(string fixturePath, string goldenPath, ILanguageAnalyzer analyzer, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(fixturePath)) throw new ArgumentException("fixturePath required", nameof(fixturePath));
if (string.IsNullOrWhiteSpace(goldenPath)) throw new ArgumentException("goldenPath required", nameof(goldenPath));
var engine = new LanguageAnalyzerEngine(new[] { analyzer });
var context = new LanguageAnalyzerContext(fixturePath, TimeProvider.System);
var result = await engine.AnalyzeAsync(context, cancellationToken).ConfigureAwait(false);
var actual = Normalize(result.ToJson(indent: true));
var expected = Normalize(await File.ReadAllTextAsync(goldenPath, cancellationToken).ConfigureAwait(false));
if (!string.Equals(actual, expected, StringComparison.Ordinal))
{
var actualPath = goldenPath + ".actual";
Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!);
await File.WriteAllTextAsync(actualPath, actual, cancellationToken).ConfigureAwait(false);
}
Assert.Equal(expected, actual);
}
private static string Normalize(string value)
{
return value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd();
}
}

View File

@@ -0,0 +1,22 @@
using System.IO;
using StellaOps.Scanner.Analyzers.Lang.Node;
using Xunit;
namespace StellaOps.Scanner.Analyzers.Lang.Node.SmokeTests;
public class Phase22SmokeTests
{
[Fact]
public async Task Phase22_Fixture_Matches_Golden()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = Path.GetFullPath(Path.Combine("..", "StellaOps.Scanner.Analyzers.Lang.Node.Tests", "Fixtures", "lang", "node", "phase22"));
var goldenPath = Path.Combine(fixturePath, "expected.json");
await LanguageAnalyzerSmokeHarness.AssertDeterministicAsync(
fixturePath,
goldenPath,
new NodeLanguageAnalyzer(),
cancellationToken);
}
}

View File

@@ -0,0 +1,32 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<IsPackable>false</IsPackable>
<!-- Stay scoped: disable implicit restore sources beyond local nugets -->
<RestoreSources>$(StellaOpsLocalNuGetSource)</RestoreSources>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="xunit.v3" Version="3.0.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.Lang/StellaOps.Scanner.Analyzers.Lang.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/StellaOps.Scanner.Analyzers.Lang.Node.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="../StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/phase22/**" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,2 @@
FROM node:22-alpine
ENV NODE_OPTIONS="--require ./bootstrap.js"

View File

@@ -0,0 +1,25 @@
[
{
"analyzerId": "node",
"componentKey": "warning:node-options:Dockerfile#2",
"purl": null,
"name": "NODE_OPTIONS warning",
"version": null,
"type": "node:warning",
"usedByEntrypoint": false,
"metadata": {
"locator": "Dockerfile#2",
"reason": "NODE_OPTIONS",
"source": "Dockerfile",
"value": "--require ./bootstrap.js"
},
"evidence": [
{
"kind": "metadata",
"source": "node.env",
"locator": "Dockerfile#2",
"value": "--require ./bootstrap.js"
}
]
}
]

View File

@@ -0,0 +1,4 @@
{
"name": "container-env",
"version": "1.0.0"
}

View File

@@ -0,0 +1,28 @@
[
{
"analyzerId": "node",
"componentKey": "purl::pkg:npm/layer-lib@0.1.0",
"purl": "pkg:npm/layer-lib@0.1.0",
"name": "layer-lib",
"version": "0.1.0",
"type": "npm",
"usedByEntrypoint": false,
"metadata": {
"entrypoint": "layers/layer1/node_modules/layer-lib/index.js",
"path": "layers/layer1/node_modules/layer-lib"
},
"evidence": [
{
"kind": "file",
"source": "package.json",
"locator": "layers/layer1/node_modules/layer-lib/package.json"
},
{
"kind": "metadata",
"source": "package.json:entrypoint",
"locator": "layers/layer1/node_modules/layer-lib/package.json#entrypoint",
"value": "layers/layer1/node_modules/layer-lib/index.js;index.js"
}
]
}
]

View File

@@ -0,0 +1,49 @@
[
{
"analyzerId": "node",
"componentKey": "observation::node-phase22",
"name": "Node Observation (Phase 22)",
"type": "node-observation",
"usedByEntrypoint": false,
"metadata": {
"node.observation.components": "1",
"node.observation.edges": "0",
"node.observation.entrypoints": "1"
},
"evidence": [
{
"kind": "derived",
"source": "node.observation",
"locator": "phase22.ndjson",
"value": "{\u0022type\u0022:\u0022component\u0022,\u0022componentType\u0022:\u0022pkg\u0022,\u0022path\u0022:\u0022/original.ts\u0022,\u0022format\u0022:\u0022esm\u0022,\u0022fromBundle\u0022:true,\u0022reason\u0022:\u0022source-map\u0022,\u0022confidence\u0022:0.87,\u0022resolverTrace\u0022:[\u0022bundle:/src/index.js\u0022,\u0022map:/src/index.js.map\u0022,\u0022source:/original.ts\u0022]}\n{\u0022type\u0022:\u0022entrypoint\u0022,\u0022path\u0022:\u0022/src/index.js\u0022,\u0022format\u0022:\u0022esm\u0022,\u0022reason\u0022:\u0022bundle-entrypoint\u0022,\u0022confidence\u0022:0.88,\u0022resolverTrace\u0022:[\u0022bundle:/src/index.js\u0022,\u0022map:/src/index.js.map\u0022]}",
"sha256": "b2d6ac4c2b422ab26943dab38c2a7b8e8fa2979122e0c2674adb5a48f9cdd2fb"
}
]
},
{
"analyzerId": "node",
"componentKey": "purl::pkg:npm/dynamic-imports@1.0.0",
"purl": "pkg:npm/dynamic-imports@1.0.0",
"name": "dynamic-imports",
"version": "1.0.0",
"type": "npm",
"usedByEntrypoint": false,
"metadata": {
"entrypoint": "src/index.js",
"path": "."
},
"evidence": [
{
"kind": "file",
"source": "package.json",
"locator": "package.json"
},
{
"kind": "metadata",
"source": "package.json:entrypoint",
"locator": "package.json#entrypoint",
"value": "src/index.js;src/index.js"
}
]
}
]

View File

@@ -0,0 +1,5 @@
{
"name": "dynamic-imports",
"version": "1.0.0",
"main": "src/index.js"
}

View File

@@ -0,0 +1,9 @@
import staticDep from './lib/static.js';
const concat = require('./lib/' + 'concat.js');
async function load(name) {
const mod = await import(`./lib/${name}/entry.js`);
return mod;
}
//# sourceMappingURL=index.js.map

View File

@@ -0,0 +1,7 @@
{
"version": 3,
"file": "index.js",
"sources": ["original.ts"],
"sourcesContent": ["import mapped from './lib/sourcemap.js';"],
"mappings": ""
}

View File

@@ -0,0 +1,27 @@
[
{
"analyzerId": "node",
"componentKey": "observation::node-phase22",
"purl": null,
"name": "Node Observation (Phase 22)",
"version": null,
"type": "node-observation",
"usedByEntrypoint": false,
"metadata": {
"node.observation.components": "3",
"node.observation.edges": "3",
"node.observation.entrypoints": "1",
"node.observation.native": "1",
"node.observation.wasm": "1"
},
"evidence": [
{
"kind": "derived",
"source": "node.observation",
"locator": "phase22.ndjson",
"value": "{\"type\":\"component\",\"componentType\":\"native\",\"path\":\"/native/addon.node\",\"reason\":\"native-addon-file\",\"confidence\":0.82,\"resolverTrace\":[\"file:/native/addon.node\"],\"arch\":\"x86_64\",\"platform\":\"linux\"}\n{\"type\":\"component\",\"componentType\":\"wasm\",\"path\":\"/pkg/pkg.wasm\",\"reason\":\"wasm-file\",\"confidence\":0.8,\"resolverTrace\":[\"file:/pkg/pkg.wasm\"]}\n{\"type\":\"component\",\"componentType\":\"pkg\",\"path\":\"/src/app.js\",\"format\":\"esm\",\"fromBundle\":true,\"reason\":\"source-map\",\"confidence\":0.87,\"resolverTrace\":[\"bundle:/dist/main.js\",\"map:/dist/main.js.map\",\"source:/src/app.js\"]}\n{\"type\":\"edge\",\"edgeType\":\"native-addon\",\"from\":\"/dist/main.js\",\"to\":\"/native/addon.node\",\"reason\":\"native-dlopen-string\",\"confidence\":0.76,\"resolverTrace\":[\"source:/dist/main.js\",\"call:process.dlopen('../native/addon.node')\"]}\n{\"type\":\"edge\",\"edgeType\":\"wasm\",\"from\":\"/dist/main.js\",\"to\":\"/pkg/pkg.wasm\",\"reason\":\"wasm-import\",\"confidence\":0.74,\"resolverTrace\":[\"source:/dist/main.js\",\"call:WebAssembly.instantiate('../pkg/pkg.wasm')\"]}\n{\"type\":\"edge\",\"edgeType\":\"capability\",\"from\":\"/dist/main.js\",\"to\":\"child_process.execFile\",\"reason\":\"capability-child-process\",\"confidence\":0.7,\"resolverTrace\":[\"source:/dist/main.js\",\"call:child_process.execFile\"]}\n{\"type\":\"entrypoint\",\"path\":\"/dist/main.js\",\"format\":\"esm\",\"reason\":\"bundle-entrypoint\",\"confidence\":0.88,\"resolverTrace\":[\"bundle:/dist/main.js\",\"map:/dist/main.js.map\"]}",
"sha256": "7e99e8fbd63eb2f29717ce6b03dc148d969b203e10a072d1bcd6ff0c5fe424bb"
}
]
}
]

View File

@@ -0,0 +1,5 @@
import childProcess from 'child_process';
export function start() {
childProcess.execFile('ls');
return WebAssembly.instantiateStreaming(fetch('./pkg/pkg.wasm'));
}

View File

@@ -0,0 +1,47 @@
[
{
"analyzerId": "node",
"componentKey": "purl::pkg:npm/pkg@1.2.3",
"purl": "pkg:npm/pkg@1.2.3",
"name": "pkg",
"version": "1.2.3",
"type": "npm",
"usedByEntrypoint": false,
"metadata": {
"entrypoint": "node_modules/.pnpm/pkg@1.2.3/node_modules/pkg/index.js",
"path": "node_modules/.pnpm/pkg@1.2.3/node_modules/pkg"
},
"evidence": [
{
"kind": "file",
"source": "package.json",
"locator": "node_modules/.pnpm/pkg@1.2.3/node_modules/pkg/package.json"
},
{
"kind": "metadata",
"source": "package.json:entrypoint",
"locator": "node_modules/.pnpm/pkg@1.2.3/node_modules/pkg/package.json#entrypoint",
"value": "node_modules/.pnpm/pkg@1.2.3/node_modules/pkg/index.js;index.js"
}
]
},
{
"analyzerId": "node",
"componentKey": "purl::pkg:npm/pnpm-demo@0.0.1",
"purl": "pkg:npm/pnpm-demo@0.0.1",
"name": "pnpm-demo",
"version": "0.0.1",
"type": "npm",
"usedByEntrypoint": false,
"metadata": {
"path": "."
},
"evidence": [
{
"kind": "file",
"source": "package.json",
"locator": "package.json"
}
]
}
]

Some files were not shown because too many files have changed in this diff Show More