up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-11-26 07:47:08 +02:00
parent 56e2f64d07
commit 1c782897f7
184 changed files with 8991 additions and 649 deletions

View File

@@ -31,6 +31,15 @@ public sealed class ReachabilityFactDocument
[BsonIgnoreIfNull]
public Dictionary<string, string?>? Metadata { get; set; }
[BsonElement("score")]
public double Score { get; set; }
[BsonElement("unknownsCount")]
public int UnknownsCount { get; set; }
[BsonElement("unknownsPressure")]
public double UnknownsPressure { get; set; }
[BsonElement("computedAt")]
public DateTimeOffset ComputedAt { get; set; }
@@ -50,6 +59,15 @@ public sealed class ReachabilityStateDocument
[BsonElement("confidence")]
public double Confidence { get; set; }
[BsonElement("bucket")]
public string Bucket { get; set; } = "unknown";
[BsonElement("weight")]
public double Weight { get; set; }
[BsonElement("score")]
public double Score { get; set; }
[BsonElement("path")]
public List<string> Path { get; set; } = new();

View File

@@ -10,4 +10,12 @@ public sealed record ReachabilityFactUpdatedEvent(
int ReachableCount,
int UnreachableCount,
int RuntimeFactsCount,
DateTimeOffset ComputedAtUtc);
string Bucket,
double Weight,
int StateCount,
double FactScore,
int UnknownsCount,
double UnknownsPressure,
double AverageConfidence,
DateTimeOffset ComputedAtUtc,
string[] Targets);

View File

@@ -0,0 +1,47 @@
using System;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Signals.Models;
public sealed class UnknownSymbolDocument
{
[BsonId]
[BsonRepresentation(BsonType.ObjectId)]
public string Id { get; set; } = ObjectId.GenerateNewId().ToString();
[BsonElement("subjectKey")]
[BsonRequired]
public string SubjectKey { get; set; } = string.Empty;
[BsonElement("callgraphId")]
[BsonIgnoreIfNull]
public string? CallgraphId { get; set; }
[BsonElement("symbolId")]
[BsonIgnoreIfNull]
public string? SymbolId { get; set; }
[BsonElement("codeId")]
[BsonIgnoreIfNull]
public string? CodeId { get; set; }
[BsonElement("purl")]
[BsonIgnoreIfNull]
public string? Purl { get; set; }
[BsonElement("edgeFrom")]
[BsonIgnoreIfNull]
public string? EdgeFrom { get; set; }
[BsonElement("edgeTo")]
[BsonIgnoreIfNull]
public string? EdgeTo { get; set; }
[BsonElement("reason")]
[BsonIgnoreIfNull]
public string? Reason { get; set; }
[BsonElement("createdAt")]
public DateTimeOffset CreatedAt { get; set; }
}

View File

@@ -0,0 +1,32 @@
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Signals.Models;
public sealed class UnknownsIngestRequest
{
[Required]
public ReachabilitySubject? Subject { get; set; }
[Required]
public string CallgraphId { get; set; } = string.Empty;
[Required]
public List<UnknownSymbolEntry> Unknowns { get; set; } = new();
}
public sealed class UnknownSymbolEntry
{
public string? SymbolId { get; set; }
public string? CodeId { get; set; }
public string? Purl { get; set; }
public string? EdgeFrom { get; set; }
public string? EdgeTo { get; set; }
public string? Reason { get; set; }
}
public sealed class UnknownsIngestResponse
{
public string SubjectKey { get; init; } = string.Empty;
public int UnknownsCount { get; init; }
}

View File

@@ -26,6 +26,11 @@ public sealed class SignalsMongoOptions
/// Collection name storing reachability facts.
/// </summary>
public string ReachabilityFactsCollection { get; set; } = "reachability_facts";
/// <summary>
/// Collection name storing unresolved symbols/edges (Unknowns Registry).
/// </summary>
public string UnknownsCollection { get; set; } = "unknowns";
/// <summary>
/// Validates the configured values.
@@ -51,5 +56,10 @@ public sealed class SignalsMongoOptions
{
throw new InvalidOperationException("Signals reachability fact collection name must be configured.");
}
if (string.IsNullOrWhiteSpace(UnknownsCollection))
{
throw new InvalidOperationException("Signals unknowns collection name must be configured.");
}
}
}

View File

@@ -32,6 +32,24 @@ public sealed class SignalsScoringOptions
/// </summary>
public double MinConfidence { get; set; } = 0.05;
/// <summary>
/// Maximum fraction to subtract from overall fact score when unknowns are present.
/// </summary>
public double UnknownsPenaltyCeiling { get; set; } = 0.35;
/// <summary>
/// Multipliers applied per reachability bucket. Keys are case-insensitive.
/// Defaults mirror policy scoring config guidance in docs/11_DATA_SCHEMAS.md.
/// </summary>
public Dictionary<string, double> ReachabilityBuckets { get; } = new(StringComparer.OrdinalIgnoreCase)
{
{ "entrypoint", 1.0 },
{ "direct", 0.85 },
{ "runtime", 0.45 },
{ "unknown", 0.5 },
{ "unreachable", 0.0 }
};
public void Validate()
{
EnsurePercent(nameof(ReachableConfidence), ReachableConfidence);
@@ -39,6 +57,11 @@ public sealed class SignalsScoringOptions
EnsurePercent(nameof(RuntimeBonus), RuntimeBonus);
EnsurePercent(nameof(MaxConfidence), MaxConfidence);
EnsurePercent(nameof(MinConfidence), MinConfidence);
EnsurePercent(nameof(UnknownsPenaltyCeiling), UnknownsPenaltyCeiling);
foreach (var (key, value) in ReachabilityBuckets)
{
EnsurePercent($"ReachabilityBuckets[{key}]", value);
}
if (MinConfidence > UnreachableConfidence)
{

View File

@@ -0,0 +1,13 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Persistence;
public interface IUnknownsRepository
{
Task UpsertAsync(string subjectKey, IEnumerable<UnknownSymbolDocument> items, CancellationToken cancellationToken);
Task<IReadOnlyList<UnknownSymbolDocument>> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken);
Task<int> CountBySubjectAsync(string subjectKey, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,53 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Persistence;
public sealed class MongoUnknownsRepository : IUnknownsRepository
{
private readonly IMongoCollection<UnknownSymbolDocument> collection;
public MongoUnknownsRepository(IMongoCollection<UnknownSymbolDocument> collection)
{
this.collection = collection ?? throw new ArgumentNullException(nameof(collection));
}
public async Task UpsertAsync(string subjectKey, IEnumerable<UnknownSymbolDocument> items, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
ArgumentNullException.ThrowIfNull(items);
// deterministic replace per subject to keep the registry stable
await collection.DeleteManyAsync(doc => doc.SubjectKey == subjectKey, cancellationToken).ConfigureAwait(false);
var batch = items.ToList();
if (batch.Count == 0)
{
return;
}
await collection.InsertManyAsync(batch, cancellationToken: cancellationToken).ConfigureAwait(false);
}
public async Task<IReadOnlyList<UnknownSymbolDocument>> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
var cursor = await collection.FindAsync(doc => doc.SubjectKey == subjectKey, cancellationToken: cancellationToken).ConfigureAwait(false);
return await cursor.ToListAsync(cancellationToken).ConfigureAwait(false);
}
public async Task<int> CountBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
var count = await collection.CountDocumentsAsync(doc => doc.SubjectKey == subjectKey, cancellationToken: cancellationToken).ConfigureAwait(false);
return (int)count;
}
}

View File

@@ -114,6 +114,15 @@ builder.Services.AddSingleton<IMongoCollection<ReachabilityFactDocument>>(sp =>
return collection;
});
builder.Services.AddSingleton<IMongoCollection<UnknownSymbolDocument>>(sp =>
{
var opts = sp.GetRequiredService<IOptions<SignalsOptions>>().Value;
var database = sp.GetRequiredService<IMongoDatabase>();
var collection = database.GetCollection<UnknownSymbolDocument>(opts.Mongo.UnknownsCollection);
EnsureUnknownsIndexes(collection);
return collection;
});
builder.Services.AddSingleton<ICallgraphRepository, MongoCallgraphRepository>();
builder.Services.AddSingleton<ICallgraphArtifactStore, FileSystemCallgraphArtifactStore>();
builder.Services.AddSingleton<ICallgraphParser>(new SimpleJsonCallgraphParser("java"));
@@ -137,6 +146,9 @@ builder.Services.AddSingleton<IReachabilityFactRepository>(sp =>
});
builder.Services.AddSingleton<IReachabilityScoringService, ReachabilityScoringService>();
builder.Services.AddSingleton<IRuntimeFactsIngestionService, RuntimeFactsIngestionService>();
builder.Services.AddSingleton<IReachabilityUnionIngestionService, ReachabilityUnionIngestionService>();
builder.Services.AddSingleton<IUnknownsRepository, MongoUnknownsRepository>();
builder.Services.AddSingleton<IUnknownsIngestionService, UnknownsIngestionService>();
if (bootstrap.Authority.Enabled)
{
@@ -392,6 +404,109 @@ signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
}
}).WithName("SignalsRuntimeIngest");
signalsGroup.MapPost("/reachability/union", async Task<IResult> (
HttpContext context,
SignalsOptions options,
[FromHeader(Name = "X-Analysis-Id")] string? analysisId,
IReachabilityUnionIngestionService ingestionService,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
{
return authFailure ?? Results.Unauthorized();
}
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
}
var id = string.IsNullOrWhiteSpace(analysisId) ? Guid.NewGuid().ToString("N") : analysisId.Trim();
if (!string.Equals(context.Request.ContentType, "application/zip", StringComparison.OrdinalIgnoreCase))
{
return Results.BadRequest(new { error = "Content-Type must be application/zip" });
}
try
{
var response = await ingestionService.IngestAsync(id, context.Request.Body, cancellationToken).ConfigureAwait(false);
return Results.Accepted($"/signals/reachability/union/{response.AnalysisId}/meta", response);
}
catch (Exception ex)
{
return Results.BadRequest(new { error = ex.Message });
}
}).WithName("SignalsReachabilityUnionIngest");
signalsGroup.MapGet("/reachability/union/{analysisId}/meta", async Task<IResult> (
HttpContext context,
SignalsOptions options,
string analysisId,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
{
return authFailure ?? Results.Unauthorized();
}
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
}
if (string.IsNullOrWhiteSpace(analysisId))
{
return Results.BadRequest(new { error = "analysisId is required." });
}
var path = Path.Combine(options.Storage.RootPath, "reachability_graphs", analysisId.Trim(), "meta.json");
if (!File.Exists(path))
{
return Results.NotFound();
}
var bytes = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
return Results.File(bytes, "application/json");
}).WithName("SignalsReachabilityUnionMeta");
signalsGroup.MapGet("/reachability/union/{analysisId}/files/{fileName}", async Task<IResult> (
HttpContext context,
SignalsOptions options,
string analysisId,
string fileName,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
{
return authFailure ?? Results.Unauthorized();
}
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
}
if (string.IsNullOrWhiteSpace(analysisId) || string.IsNullOrWhiteSpace(fileName))
{
return Results.BadRequest(new { error = "analysisId and fileName are required." });
}
var root = Path.Combine(options.Storage.RootPath, "reachability_graphs", analysisId.Trim());
var path = Path.Combine(root, fileName.Replace('/', Path.DirectorySeparatorChar));
if (!File.Exists(path))
{
return Results.NotFound();
}
var contentType = fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ? "application/json" : "application/x-ndjson";
var bytes = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
return Results.File(bytes, contentType);
}).WithName("SignalsReachabilityUnionFile");
signalsGroup.MapPost("/runtime-facts/ndjson", async Task<IResult> (
HttpContext context,
SignalsOptions options,
@@ -469,6 +584,62 @@ signalsGroup.MapGet("/facts/{subjectKey}", async Task<IResult> (
return fact is null ? Results.NotFound() : Results.Ok(fact);
}).WithName("SignalsFactsGet");
signalsGroup.MapPost("/unknowns", async Task<IResult> (
HttpContext context,
SignalsOptions options,
UnknownsIngestRequest request,
IUnknownsIngestionService ingestionService,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
{
return authFailure ?? Results.Unauthorized();
}
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
}
try
{
var response = await ingestionService.IngestAsync(request, cancellationToken).ConfigureAwait(false);
return Results.Accepted($"/signals/unknowns/{response.SubjectKey}", response);
}
catch (UnknownsValidationException ex)
{
return Results.BadRequest(new { error = ex.Message });
}
}).WithName("SignalsUnknownsIngest");
signalsGroup.MapGet("/unknowns/{subjectKey}", async Task<IResult> (
HttpContext context,
SignalsOptions options,
string subjectKey,
IUnknownsRepository repository,
SignalsSealedModeMonitor sealedModeMonitor,
CancellationToken cancellationToken) =>
{
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
{
return authFailure ?? Results.Unauthorized();
}
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
{
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
}
if (string.IsNullOrWhiteSpace(subjectKey))
{
return Results.BadRequest(new { error = "subjectKey is required." });
}
var items = await repository.GetBySubjectAsync(subjectKey.Trim(), cancellationToken).ConfigureAwait(false);
return items.Count == 0 ? Results.NotFound() : Results.Ok(items);
}).WithName("SignalsUnknownsGet");
signalsGroup.MapPost("/reachability/recompute", async Task<IResult> (
HttpContext context,
SignalsOptions options,
@@ -621,4 +792,31 @@ public partial class Program
statusCode: StatusCodes.Status503ServiceUnavailable);
return false;
}
internal static void EnsureUnknownsIndexes(IMongoCollection<UnknownSymbolDocument> collection)
{
ArgumentNullException.ThrowIfNull(collection);
try
{
var subjectIndex = new CreateIndexModel<UnknownSymbolDocument>(
Builders<UnknownSymbolDocument>.IndexKeys.Ascending(doc => doc.SubjectKey),
new CreateIndexOptions { Name = "unknowns_subject_lookup" });
var dedupeIndex = new CreateIndexModel<UnknownSymbolDocument>(
Builders<UnknownSymbolDocument>.IndexKeys
.Ascending(doc => doc.SubjectKey)
.Ascending(doc => doc.SymbolId)
.Ascending(doc => doc.Purl)
.Ascending(doc => doc.EdgeFrom)
.Ascending(doc => doc.EdgeTo),
new CreateIndexOptions { Name = "unknowns_subject_symbol_edge_unique", Unique = true });
collection.Indexes.CreateMany(new[] { subjectIndex, dedupeIndex });
}
catch (MongoCommandException ex) when (string.Equals(ex.CodeName, "IndexOptionsConflict", StringComparison.Ordinal))
{
// Ignore to keep startup idempotent when index options differ.
}
}
}

View File

@@ -5,5 +5,5 @@ namespace StellaOps.Signals.Services;
public interface IEventsPublisher
{
Task PublishFactUpdatedAsync(Models.ReachabilityFactDocument fact, CancellationToken cancellationToken);
Task PublishFactUpdatedAsync(global::StellaOps.Signals.Models.ReachabilityFactDocument fact, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,14 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Signals.Services.Models;
namespace StellaOps.Signals.Services;
/// <summary>
/// Ingests runtime+static union bundles and normalizes them into the reachability CAS layout.
/// </summary>
public interface IReachabilityUnionIngestionService
{
Task<ReachabilityUnionIngestResponse> IngestAsync(string analysisId, Stream zipStream, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,10 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Services;
public interface IUnknownsIngestionService
{
Task<UnknownsIngestResponse> IngestAsync(UnknownsIngestRequest request, CancellationToken cancellationToken);
}

View File

@@ -31,6 +31,17 @@ internal sealed class InMemoryEventsPublisher : IEventsPublisher
var (reachable, unreachable) = CountStates(fact);
var runtimeFactsCount = fact.RuntimeFacts?.Count ?? 0;
var avgConfidence = fact.States.Count > 0 ? fact.States.Average(s => s.Confidence) : 0;
var score = fact.Score;
var unknownsCount = fact.UnknownsCount;
var unknownsPressure = fact.UnknownsPressure;
var topBucket = fact.States.Count > 0
? fact.States
.GroupBy(s => s.Bucket, StringComparer.OrdinalIgnoreCase)
.OrderByDescending(g => g.Count())
.ThenByDescending(g => g.Average(s => s.Weight))
.First()
: null;
var payload = new ReachabilityFactUpdatedEvent(
Version: "signals.fact.updated@v1",
SubjectKey: fact.SubjectKey,
@@ -39,7 +50,15 @@ internal sealed class InMemoryEventsPublisher : IEventsPublisher
ReachableCount: reachable,
UnreachableCount: unreachable,
RuntimeFactsCount: runtimeFactsCount,
ComputedAtUtc: fact.ComputedAt);
Bucket: topBucket?.Key ?? "unknown",
Weight: topBucket?.Average(s => s.Weight) ?? 0,
StateCount: fact.States.Count,
FactScore: score,
UnknownsCount: unknownsCount,
UnknownsPressure: unknownsPressure,
AverageConfidence: avgConfidence,
ComputedAtUtc: fact.ComputedAt,
Targets: fact.States.Select(s => s.Target).ToArray());
var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web));
logger.LogInformation("{Topic} {Payload}", topic, json);

View File

@@ -0,0 +1,13 @@
using System.Collections.Generic;
namespace StellaOps.Signals.Services.Models;
public sealed record ReachabilityUnionIngestResponse(
string AnalysisId,
string CasRoot,
IReadOnlyList<ReachabilityUnionFile> Files);
public sealed record ReachabilityUnionFile(
string Path,
string Sha256,
int? Records);

View File

@@ -18,6 +18,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
private readonly TimeProvider timeProvider;
private readonly SignalsScoringOptions scoringOptions;
private readonly IReachabilityCache cache;
private readonly IUnknownsRepository unknownsRepository;
private readonly IEventsPublisher eventsPublisher;
private readonly ILogger<ReachabilityScoringService> logger;
@@ -27,6 +28,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
TimeProvider timeProvider,
IOptions<SignalsOptions> options,
IReachabilityCache cache,
IUnknownsRepository unknownsRepository,
IEventsPublisher eventsPublisher,
ILogger<ReachabilityScoringService> logger)
{
@@ -35,6 +37,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.scoringOptions = options?.Value?.Scoring ?? throw new ArgumentNullException(nameof(options));
this.cache = cache ?? throw new ArgumentNullException(nameof(cache));
this.unknownsRepository = unknownsRepository ?? throw new ArgumentNullException(nameof(unknownsRepository));
this.eventsPublisher = eventsPublisher ?? throw new ArgumentNullException(nameof(eventsPublisher));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
@@ -94,22 +97,25 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
{
var path = FindPath(entryPoints, target, graph.Adjacency);
var reachable = path is not null;
var confidence = reachable ? scoringOptions.ReachableConfidence : scoringOptions.UnreachableConfidence;
var runtimeEvidence = runtimeHits.Where(hit => path?.Contains(hit, StringComparer.Ordinal) == true).ToList();
var runtimeEvidence = runtimeHits.Where(hit => path?.Contains(hit, StringComparer.Ordinal) == true)
.ToList();
if (runtimeEvidence.Count > 0)
{
confidence = Math.Min(scoringOptions.MaxConfidence, confidence + scoringOptions.RuntimeBonus);
}
var (bucket, weight, confidence) = ComputeScores(
reachable,
entryPoints,
target,
path,
runtimeEvidence.Count);
confidence = Math.Clamp(confidence, scoringOptions.MinConfidence, scoringOptions.MaxConfidence);
var score = confidence * weight;
states.Add(new ReachabilityStateDocument
{
Target = target,
Reachable = reachable,
Confidence = confidence,
Bucket = bucket,
Weight = weight,
Score = score,
Path = path ?? new List<string>(),
Evidence = new ReachabilityEvidenceDocument
{
@@ -119,6 +125,14 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
});
}
var baseScore = states.Count > 0 ? states.Average(s => s.Score) : 0;
var unknownsCount = await unknownsRepository.CountBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
var pressure = states.Count + unknownsCount == 0
? 0
: Math.Min(1.0, Math.Max(0.0, unknownsCount / (double)(states.Count + unknownsCount)));
var pressurePenalty = Math.Min(scoringOptions.UnknownsPenaltyCeiling, pressure);
var finalScore = baseScore * (1 - pressurePenalty);
var document = new ReachabilityFactDocument
{
CallgraphId = request.CallgraphId,
@@ -126,6 +140,9 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
EntryPoints = entryPoints,
States = states,
Metadata = request.Metadata,
Score = finalScore,
UnknownsCount = unknownsCount,
UnknownsPressure = pressure,
ComputedAt = timeProvider.GetUtcNow(),
SubjectKey = subjectKey,
RuntimeFacts = existingFact?.RuntimeFacts
@@ -278,6 +295,51 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
return path;
}
private (string bucket, double weight, double confidence) ComputeScores(
bool reachable,
List<string> entryPoints,
string target,
List<string>? path,
int runtimeEvidenceCount)
{
var bucket = "unknown";
if (!reachable)
{
bucket = "unreachable";
}
else if (entryPoints.Contains(target, StringComparer.Ordinal))
{
bucket = "entrypoint";
}
else if (runtimeEvidenceCount > 0)
{
bucket = "runtime";
}
else if (path is not null && path.Count <= 2)
{
bucket = "direct";
}
else
{
bucket = "unknown";
}
var weight = scoringOptions.ReachabilityBuckets.TryGetValue(bucket, out var w)
? w
: scoringOptions.ReachabilityBuckets.TryGetValue("unknown", out var unknown)
? unknown
: 1.0;
var confidence = reachable ? scoringOptions.ReachableConfidence : scoringOptions.UnreachableConfidence;
if (runtimeEvidenceCount > 0 && reachable)
{
confidence = Math.Min(scoringOptions.MaxConfidence, confidence + scoringOptions.RuntimeBonus);
}
confidence = Math.Clamp(confidence, scoringOptions.MinConfidence, scoringOptions.MaxConfidence);
return (bucket, weight, confidence);
}
private sealed record ReachabilityGraph(
HashSet<string> Nodes,
Dictionary<string, HashSet<string>> Adjacency,

View File

@@ -0,0 +1,126 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
using StellaOps.Signals.Services.Models;
namespace StellaOps.Signals.Services;
/// <summary>
/// Writes reachability union bundles (runtime + static) into the CAS layout: reachability_graphs/&lt;analysisId&gt;/
/// Validates meta.json hashes before persisting.
/// </summary>
public sealed class ReachabilityUnionIngestionService : IReachabilityUnionIngestionService
{
private static readonly string[] RequiredFiles = { "nodes.ndjson", "edges.ndjson", "meta.json" };
private readonly ILogger<ReachabilityUnionIngestionService> logger;
private readonly SignalsOptions options;
public ReachabilityUnionIngestionService(
ILogger<ReachabilityUnionIngestionService> logger,
IOptions<SignalsOptions> options)
{
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
}
public async Task<ReachabilityUnionIngestResponse> IngestAsync(string analysisId, Stream zipStream, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(analysisId);
ArgumentNullException.ThrowIfNull(zipStream);
var casRoot = Path.Combine(options.Storage.RootPath, "reachability_graphs", analysisId.Trim());
if (Directory.Exists(casRoot))
{
Directory.Delete(casRoot, recursive: true);
}
Directory.CreateDirectory(casRoot);
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read, leaveOpen: true);
var entries = archive.Entries.ToDictionary(e => e.FullName, StringComparer.OrdinalIgnoreCase);
foreach (var required in RequiredFiles)
{
if (!entries.ContainsKey(required))
{
throw new InvalidOperationException($"Union bundle missing required file: {required}");
}
}
var metaEntry = entries["meta.json"];
using var metaStream = metaEntry.Open();
using var metaDoc = await JsonDocument.ParseAsync(metaStream, cancellationToken: cancellationToken).ConfigureAwait(false);
var metaRoot = metaDoc.RootElement;
var filesElement = metaRoot.TryGetProperty("files", out var f) && f.ValueKind == JsonValueKind.Array
? f
: throw new InvalidOperationException("meta.json is missing required 'files' array");
var recorded = filesElement.EnumerateArray()
.Select(el => new
{
Path = el.GetProperty("path").GetString() ?? string.Empty,
Sha = el.GetProperty("sha256").GetString() ?? string.Empty,
Records = el.TryGetProperty("records", out var r) && r.ValueKind == JsonValueKind.Number ? r.GetInt32() : (int?)null
})
.ToList();
var filesForResponse = new List<ReachabilityUnionFile>();
foreach (var file in recorded)
{
if (!entries.TryGetValue(file.Path, out var zipEntry))
{
throw new InvalidOperationException($"meta.json references missing file '{file.Path}'.");
}
var destPath = Path.Combine(casRoot, file.Path.Replace('/', Path.DirectorySeparatorChar));
Directory.CreateDirectory(Path.GetDirectoryName(destPath)!);
using (var entryStream = zipEntry.Open())
using (var dest = File.Create(destPath))
{
await entryStream.CopyToAsync(dest, cancellationToken).ConfigureAwait(false);
}
var actualSha = ComputeSha256Hex(destPath);
if (!string.Equals(actualSha, file.Sha, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException($"SHA mismatch for {file.Path}: expected {file.Sha}, actual {actualSha}.");
}
filesForResponse.Add(new ReachabilityUnionFile(file.Path, actualSha, file.Records));
}
logger.LogInformation("Ingested reachability union bundle {AnalysisId} with {FileCount} files.", analysisId, filesForResponse.Count);
return new ReachabilityUnionIngestResponse(analysisId, $"cas://reachability_graphs/{analysisId}", filesForResponse);
}
private static string ComputeSha256Hex(string path)
{
using var stream = File.OpenRead(path);
var buffer = new byte[8192];
using var sha = SHA256.Create();
int read;
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
{
sha.TransformBlock(buffer, 0, read, null, 0);
}
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
return Convert.ToHexString(sha.Hash!).ToLowerInvariant();
}
}

View File

@@ -0,0 +1,99 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using StellaOps.Signals.Models;
using StellaOps.Signals.Persistence;
namespace StellaOps.Signals.Services;
internal sealed class UnknownsIngestionService : IUnknownsIngestionService
{
private readonly IUnknownsRepository repository;
private readonly TimeProvider timeProvider;
private readonly ILogger<UnknownsIngestionService> logger;
public UnknownsIngestionService(IUnknownsRepository repository, TimeProvider timeProvider, ILogger<UnknownsIngestionService> logger)
{
this.repository = repository ?? throw new ArgumentNullException(nameof(repository));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<UnknownsIngestResponse> IngestAsync(UnknownsIngestRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
if (request.Subject is null)
{
throw new UnknownsValidationException("Subject is required.");
}
if (string.IsNullOrWhiteSpace(request.CallgraphId))
{
throw new UnknownsValidationException("callgraphId is required.");
}
if (request.Unknowns is null || request.Unknowns.Count == 0)
{
throw new UnknownsValidationException("Unknowns list must not be empty.");
}
var subjectKey = request.Subject.ToSubjectKey();
if (string.IsNullOrWhiteSpace(subjectKey))
{
throw new UnknownsValidationException("Subject must include scanId, imageDigest, or component/version.");
}
var now = timeProvider.GetUtcNow();
var normalized = new List<UnknownSymbolDocument>();
foreach (var entry in request.Unknowns)
{
if (entry is null)
{
continue;
}
var hasContent = !(string.IsNullOrWhiteSpace(entry.SymbolId)
&& string.IsNullOrWhiteSpace(entry.CodeId)
&& string.IsNullOrWhiteSpace(entry.Purl)
&& string.IsNullOrWhiteSpace(entry.EdgeFrom)
&& string.IsNullOrWhiteSpace(entry.EdgeTo));
if (!hasContent)
{
continue;
}
normalized.Add(new UnknownSymbolDocument
{
SubjectKey = subjectKey,
CallgraphId = request.CallgraphId,
SymbolId = entry.SymbolId?.Trim(),
CodeId = entry.CodeId?.Trim(),
Purl = entry.Purl?.Trim(),
EdgeFrom = entry.EdgeFrom?.Trim(),
EdgeTo = entry.EdgeTo?.Trim(),
Reason = entry.Reason?.Trim(),
CreatedAt = now
});
}
if (normalized.Count == 0)
{
throw new UnknownsValidationException("Unknown entries must include at least one symbolId, codeId, purl, or edge.");
}
await repository.UpsertAsync(subjectKey, normalized, cancellationToken).ConfigureAwait(false);
logger.LogInformation("Stored {Count} unknown symbols for subject {SubjectKey}", normalized.Count, subjectKey);
return new UnknownsIngestResponse
{
SubjectKey = subjectKey,
UnknownsCount = normalized.Count
};
}
}

View File

@@ -0,0 +1,10 @@
using System;
namespace StellaOps.Signals.Services;
public sealed class UnknownsValidationException : Exception
{
public UnknownsValidationException(string message) : base(message)
{
}
}