up
This commit is contained in:
@@ -31,6 +31,15 @@ public sealed class ReachabilityFactDocument
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
|
||||
[BsonElement("score")]
|
||||
public double Score { get; set; }
|
||||
|
||||
[BsonElement("unknownsCount")]
|
||||
public int UnknownsCount { get; set; }
|
||||
|
||||
[BsonElement("unknownsPressure")]
|
||||
public double UnknownsPressure { get; set; }
|
||||
|
||||
[BsonElement("computedAt")]
|
||||
public DateTimeOffset ComputedAt { get; set; }
|
||||
|
||||
@@ -50,6 +59,15 @@ public sealed class ReachabilityStateDocument
|
||||
[BsonElement("confidence")]
|
||||
public double Confidence { get; set; }
|
||||
|
||||
[BsonElement("bucket")]
|
||||
public string Bucket { get; set; } = "unknown";
|
||||
|
||||
[BsonElement("weight")]
|
||||
public double Weight { get; set; }
|
||||
|
||||
[BsonElement("score")]
|
||||
public double Score { get; set; }
|
||||
|
||||
[BsonElement("path")]
|
||||
public List<string> Path { get; set; } = new();
|
||||
|
||||
|
||||
@@ -10,4 +10,12 @@ public sealed record ReachabilityFactUpdatedEvent(
|
||||
int ReachableCount,
|
||||
int UnreachableCount,
|
||||
int RuntimeFactsCount,
|
||||
DateTimeOffset ComputedAtUtc);
|
||||
string Bucket,
|
||||
double Weight,
|
||||
int StateCount,
|
||||
double FactScore,
|
||||
int UnknownsCount,
|
||||
double UnknownsPressure,
|
||||
double AverageConfidence,
|
||||
DateTimeOffset ComputedAtUtc,
|
||||
string[] Targets);
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using System;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
public sealed class UnknownSymbolDocument
|
||||
{
|
||||
[BsonId]
|
||||
[BsonRepresentation(BsonType.ObjectId)]
|
||||
public string Id { get; set; } = ObjectId.GenerateNewId().ToString();
|
||||
|
||||
[BsonElement("subjectKey")]
|
||||
[BsonRequired]
|
||||
public string SubjectKey { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("callgraphId")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? CallgraphId { get; set; }
|
||||
|
||||
[BsonElement("symbolId")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? SymbolId { get; set; }
|
||||
|
||||
[BsonElement("codeId")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? CodeId { get; set; }
|
||||
|
||||
[BsonElement("purl")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? Purl { get; set; }
|
||||
|
||||
[BsonElement("edgeFrom")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? EdgeFrom { get; set; }
|
||||
|
||||
[BsonElement("edgeTo")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? EdgeTo { get; set; }
|
||||
|
||||
[BsonElement("reason")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? Reason { get; set; }
|
||||
|
||||
[BsonElement("createdAt")]
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
public sealed class UnknownsIngestRequest
|
||||
{
|
||||
[Required]
|
||||
public ReachabilitySubject? Subject { get; set; }
|
||||
|
||||
[Required]
|
||||
public string CallgraphId { get; set; } = string.Empty;
|
||||
|
||||
[Required]
|
||||
public List<UnknownSymbolEntry> Unknowns { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class UnknownSymbolEntry
|
||||
{
|
||||
public string? SymbolId { get; set; }
|
||||
public string? CodeId { get; set; }
|
||||
public string? Purl { get; set; }
|
||||
public string? EdgeFrom { get; set; }
|
||||
public string? EdgeTo { get; set; }
|
||||
public string? Reason { get; set; }
|
||||
}
|
||||
|
||||
public sealed class UnknownsIngestResponse
|
||||
{
|
||||
public string SubjectKey { get; init; } = string.Empty;
|
||||
public int UnknownsCount { get; init; }
|
||||
}
|
||||
@@ -26,6 +26,11 @@ public sealed class SignalsMongoOptions
|
||||
/// Collection name storing reachability facts.
|
||||
/// </summary>
|
||||
public string ReachabilityFactsCollection { get; set; } = "reachability_facts";
|
||||
|
||||
/// <summary>
|
||||
/// Collection name storing unresolved symbols/edges (Unknowns Registry).
|
||||
/// </summary>
|
||||
public string UnknownsCollection { get; set; } = "unknowns";
|
||||
|
||||
/// <summary>
|
||||
/// Validates the configured values.
|
||||
@@ -51,5 +56,10 @@ public sealed class SignalsMongoOptions
|
||||
{
|
||||
throw new InvalidOperationException("Signals reachability fact collection name must be configured.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(UnknownsCollection))
|
||||
{
|
||||
throw new InvalidOperationException("Signals unknowns collection name must be configured.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,24 @@ public sealed class SignalsScoringOptions
|
||||
/// </summary>
|
||||
public double MinConfidence { get; set; } = 0.05;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum fraction to subtract from overall fact score when unknowns are present.
|
||||
/// </summary>
|
||||
public double UnknownsPenaltyCeiling { get; set; } = 0.35;
|
||||
|
||||
/// <summary>
|
||||
/// Multipliers applied per reachability bucket. Keys are case-insensitive.
|
||||
/// Defaults mirror policy scoring config guidance in docs/11_DATA_SCHEMAS.md.
|
||||
/// </summary>
|
||||
public Dictionary<string, double> ReachabilityBuckets { get; } = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
{ "entrypoint", 1.0 },
|
||||
{ "direct", 0.85 },
|
||||
{ "runtime", 0.45 },
|
||||
{ "unknown", 0.5 },
|
||||
{ "unreachable", 0.0 }
|
||||
};
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
EnsurePercent(nameof(ReachableConfidence), ReachableConfidence);
|
||||
@@ -39,6 +57,11 @@ public sealed class SignalsScoringOptions
|
||||
EnsurePercent(nameof(RuntimeBonus), RuntimeBonus);
|
||||
EnsurePercent(nameof(MaxConfidence), MaxConfidence);
|
||||
EnsurePercent(nameof(MinConfidence), MinConfidence);
|
||||
EnsurePercent(nameof(UnknownsPenaltyCeiling), UnknownsPenaltyCeiling);
|
||||
foreach (var (key, value) in ReachabilityBuckets)
|
||||
{
|
||||
EnsurePercent($"ReachabilityBuckets[{key}]", value);
|
||||
}
|
||||
|
||||
if (MinConfidence > UnreachableConfidence)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Persistence;
|
||||
|
||||
public interface IUnknownsRepository
|
||||
{
|
||||
Task UpsertAsync(string subjectKey, IEnumerable<UnknownSymbolDocument> items, CancellationToken cancellationToken);
|
||||
Task<IReadOnlyList<UnknownSymbolDocument>> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken);
|
||||
Task<int> CountBySubjectAsync(string subjectKey, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Persistence;
|
||||
|
||||
public sealed class MongoUnknownsRepository : IUnknownsRepository
|
||||
{
|
||||
private readonly IMongoCollection<UnknownSymbolDocument> collection;
|
||||
|
||||
public MongoUnknownsRepository(IMongoCollection<UnknownSymbolDocument> collection)
|
||||
{
|
||||
this.collection = collection ?? throw new ArgumentNullException(nameof(collection));
|
||||
}
|
||||
|
||||
public async Task UpsertAsync(string subjectKey, IEnumerable<UnknownSymbolDocument> items, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
|
||||
ArgumentNullException.ThrowIfNull(items);
|
||||
|
||||
// deterministic replace per subject to keep the registry stable
|
||||
await collection.DeleteManyAsync(doc => doc.SubjectKey == subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var batch = items.ToList();
|
||||
if (batch.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await collection.InsertManyAsync(batch, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<UnknownSymbolDocument>> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
|
||||
|
||||
var cursor = await collection.FindAsync(doc => doc.SubjectKey == subjectKey, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
return await cursor.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<int> CountBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
|
||||
|
||||
var count = await collection.CountDocumentsAsync(doc => doc.SubjectKey == subjectKey, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
return (int)count;
|
||||
}
|
||||
}
|
||||
@@ -114,6 +114,15 @@ builder.Services.AddSingleton<IMongoCollection<ReachabilityFactDocument>>(sp =>
|
||||
return collection;
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<IMongoCollection<UnknownSymbolDocument>>(sp =>
|
||||
{
|
||||
var opts = sp.GetRequiredService<IOptions<SignalsOptions>>().Value;
|
||||
var database = sp.GetRequiredService<IMongoDatabase>();
|
||||
var collection = database.GetCollection<UnknownSymbolDocument>(opts.Mongo.UnknownsCollection);
|
||||
EnsureUnknownsIndexes(collection);
|
||||
return collection;
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<ICallgraphRepository, MongoCallgraphRepository>();
|
||||
builder.Services.AddSingleton<ICallgraphArtifactStore, FileSystemCallgraphArtifactStore>();
|
||||
builder.Services.AddSingleton<ICallgraphParser>(new SimpleJsonCallgraphParser("java"));
|
||||
@@ -137,6 +146,9 @@ builder.Services.AddSingleton<IReachabilityFactRepository>(sp =>
|
||||
});
|
||||
builder.Services.AddSingleton<IReachabilityScoringService, ReachabilityScoringService>();
|
||||
builder.Services.AddSingleton<IRuntimeFactsIngestionService, RuntimeFactsIngestionService>();
|
||||
builder.Services.AddSingleton<IReachabilityUnionIngestionService, ReachabilityUnionIngestionService>();
|
||||
builder.Services.AddSingleton<IUnknownsRepository, MongoUnknownsRepository>();
|
||||
builder.Services.AddSingleton<IUnknownsIngestionService, UnknownsIngestionService>();
|
||||
|
||||
if (bootstrap.Authority.Enabled)
|
||||
{
|
||||
@@ -392,6 +404,109 @@ signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
|
||||
}
|
||||
}).WithName("SignalsRuntimeIngest");
|
||||
|
||||
signalsGroup.MapPost("/reachability/union", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
[FromHeader(Name = "X-Analysis-Id")] string? analysisId,
|
||||
IReachabilityUnionIngestionService ingestionService,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
var id = string.IsNullOrWhiteSpace(analysisId) ? Guid.NewGuid().ToString("N") : analysisId.Trim();
|
||||
|
||||
if (!string.Equals(context.Request.ContentType, "application/zip", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return Results.BadRequest(new { error = "Content-Type must be application/zip" });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var response = await ingestionService.IngestAsync(id, context.Request.Body, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Accepted($"/signals/reachability/union/{response.AnalysisId}/meta", response);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}).WithName("SignalsReachabilityUnionIngest");
|
||||
|
||||
signalsGroup.MapGet("/reachability/union/{analysisId}/meta", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
string analysisId,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(analysisId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "analysisId is required." });
|
||||
}
|
||||
|
||||
var path = Path.Combine(options.Storage.RootPath, "reachability_graphs", analysisId.Trim(), "meta.json");
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var bytes = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
|
||||
return Results.File(bytes, "application/json");
|
||||
}).WithName("SignalsReachabilityUnionMeta");
|
||||
|
||||
signalsGroup.MapGet("/reachability/union/{analysisId}/files/{fileName}", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
string analysisId,
|
||||
string fileName,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(analysisId) || string.IsNullOrWhiteSpace(fileName))
|
||||
{
|
||||
return Results.BadRequest(new { error = "analysisId and fileName are required." });
|
||||
}
|
||||
|
||||
var root = Path.Combine(options.Storage.RootPath, "reachability_graphs", analysisId.Trim());
|
||||
var path = Path.Combine(root, fileName.Replace('/', Path.DirectorySeparatorChar));
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var contentType = fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ? "application/json" : "application/x-ndjson";
|
||||
var bytes = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
|
||||
return Results.File(bytes, contentType);
|
||||
}).WithName("SignalsReachabilityUnionFile");
|
||||
|
||||
signalsGroup.MapPost("/runtime-facts/ndjson", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
@@ -469,6 +584,62 @@ signalsGroup.MapGet("/facts/{subjectKey}", async Task<IResult> (
|
||||
return fact is null ? Results.NotFound() : Results.Ok(fact);
|
||||
}).WithName("SignalsFactsGet");
|
||||
|
||||
signalsGroup.MapPost("/unknowns", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
UnknownsIngestRequest request,
|
||||
IUnknownsIngestionService ingestionService,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var response = await ingestionService.IngestAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Accepted($"/signals/unknowns/{response.SubjectKey}", response);
|
||||
}
|
||||
catch (UnknownsValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}).WithName("SignalsUnknownsIngest");
|
||||
|
||||
signalsGroup.MapGet("/unknowns/{subjectKey}", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
string subjectKey,
|
||||
IUnknownsRepository repository,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
return Results.BadRequest(new { error = "subjectKey is required." });
|
||||
}
|
||||
|
||||
var items = await repository.GetBySubjectAsync(subjectKey.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
return items.Count == 0 ? Results.NotFound() : Results.Ok(items);
|
||||
}).WithName("SignalsUnknownsGet");
|
||||
|
||||
signalsGroup.MapPost("/reachability/recompute", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
@@ -621,4 +792,31 @@ public partial class Program
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||
return false;
|
||||
}
|
||||
|
||||
internal static void EnsureUnknownsIndexes(IMongoCollection<UnknownSymbolDocument> collection)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(collection);
|
||||
|
||||
try
|
||||
{
|
||||
var subjectIndex = new CreateIndexModel<UnknownSymbolDocument>(
|
||||
Builders<UnknownSymbolDocument>.IndexKeys.Ascending(doc => doc.SubjectKey),
|
||||
new CreateIndexOptions { Name = "unknowns_subject_lookup" });
|
||||
|
||||
var dedupeIndex = new CreateIndexModel<UnknownSymbolDocument>(
|
||||
Builders<UnknownSymbolDocument>.IndexKeys
|
||||
.Ascending(doc => doc.SubjectKey)
|
||||
.Ascending(doc => doc.SymbolId)
|
||||
.Ascending(doc => doc.Purl)
|
||||
.Ascending(doc => doc.EdgeFrom)
|
||||
.Ascending(doc => doc.EdgeTo),
|
||||
new CreateIndexOptions { Name = "unknowns_subject_symbol_edge_unique", Unique = true });
|
||||
|
||||
collection.Indexes.CreateMany(new[] { subjectIndex, dedupeIndex });
|
||||
}
|
||||
catch (MongoCommandException ex) when (string.Equals(ex.CodeName, "IndexOptionsConflict", StringComparison.Ordinal))
|
||||
{
|
||||
// Ignore to keep startup idempotent when index options differ.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,5 +5,5 @@ namespace StellaOps.Signals.Services;
|
||||
|
||||
public interface IEventsPublisher
|
||||
{
|
||||
Task PublishFactUpdatedAsync(Models.ReachabilityFactDocument fact, CancellationToken cancellationToken);
|
||||
Task PublishFactUpdatedAsync(global::StellaOps.Signals.Models.ReachabilityFactDocument fact, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Services.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Ingests runtime+static union bundles and normalizes them into the reachability CAS layout.
|
||||
/// </summary>
|
||||
public interface IReachabilityUnionIngestionService
|
||||
{
|
||||
Task<ReachabilityUnionIngestResponse> IngestAsync(string analysisId, Stream zipStream, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
public interface IUnknownsIngestionService
|
||||
{
|
||||
Task<UnknownsIngestResponse> IngestAsync(UnknownsIngestRequest request, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -31,6 +31,17 @@ internal sealed class InMemoryEventsPublisher : IEventsPublisher
|
||||
|
||||
var (reachable, unreachable) = CountStates(fact);
|
||||
var runtimeFactsCount = fact.RuntimeFacts?.Count ?? 0;
|
||||
var avgConfidence = fact.States.Count > 0 ? fact.States.Average(s => s.Confidence) : 0;
|
||||
var score = fact.Score;
|
||||
var unknownsCount = fact.UnknownsCount;
|
||||
var unknownsPressure = fact.UnknownsPressure;
|
||||
var topBucket = fact.States.Count > 0
|
||||
? fact.States
|
||||
.GroupBy(s => s.Bucket, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderByDescending(g => g.Count())
|
||||
.ThenByDescending(g => g.Average(s => s.Weight))
|
||||
.First()
|
||||
: null;
|
||||
var payload = new ReachabilityFactUpdatedEvent(
|
||||
Version: "signals.fact.updated@v1",
|
||||
SubjectKey: fact.SubjectKey,
|
||||
@@ -39,7 +50,15 @@ internal sealed class InMemoryEventsPublisher : IEventsPublisher
|
||||
ReachableCount: reachable,
|
||||
UnreachableCount: unreachable,
|
||||
RuntimeFactsCount: runtimeFactsCount,
|
||||
ComputedAtUtc: fact.ComputedAt);
|
||||
Bucket: topBucket?.Key ?? "unknown",
|
||||
Weight: topBucket?.Average(s => s.Weight) ?? 0,
|
||||
StateCount: fact.States.Count,
|
||||
FactScore: score,
|
||||
UnknownsCount: unknownsCount,
|
||||
UnknownsPressure: unknownsPressure,
|
||||
AverageConfidence: avgConfidence,
|
||||
ComputedAtUtc: fact.ComputedAt,
|
||||
Targets: fact.States.Select(s => s.Target).ToArray());
|
||||
|
||||
var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web));
|
||||
logger.LogInformation("{Topic} {Payload}", topic, json);
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Signals.Services.Models;
|
||||
|
||||
public sealed record ReachabilityUnionIngestResponse(
|
||||
string AnalysisId,
|
||||
string CasRoot,
|
||||
IReadOnlyList<ReachabilityUnionFile> Files);
|
||||
|
||||
public sealed record ReachabilityUnionFile(
|
||||
string Path,
|
||||
string Sha256,
|
||||
int? Records);
|
||||
@@ -18,6 +18,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly SignalsScoringOptions scoringOptions;
|
||||
private readonly IReachabilityCache cache;
|
||||
private readonly IUnknownsRepository unknownsRepository;
|
||||
private readonly IEventsPublisher eventsPublisher;
|
||||
private readonly ILogger<ReachabilityScoringService> logger;
|
||||
|
||||
@@ -27,6 +28,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
TimeProvider timeProvider,
|
||||
IOptions<SignalsOptions> options,
|
||||
IReachabilityCache cache,
|
||||
IUnknownsRepository unknownsRepository,
|
||||
IEventsPublisher eventsPublisher,
|
||||
ILogger<ReachabilityScoringService> logger)
|
||||
{
|
||||
@@ -35,6 +37,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
this.scoringOptions = options?.Value?.Scoring ?? throw new ArgumentNullException(nameof(options));
|
||||
this.cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
this.unknownsRepository = unknownsRepository ?? throw new ArgumentNullException(nameof(unknownsRepository));
|
||||
this.eventsPublisher = eventsPublisher ?? throw new ArgumentNullException(nameof(eventsPublisher));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
@@ -94,22 +97,25 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
{
|
||||
var path = FindPath(entryPoints, target, graph.Adjacency);
|
||||
var reachable = path is not null;
|
||||
var confidence = reachable ? scoringOptions.ReachableConfidence : scoringOptions.UnreachableConfidence;
|
||||
var runtimeEvidence = runtimeHits.Where(hit => path?.Contains(hit, StringComparer.Ordinal) == true).ToList();
|
||||
|
||||
var runtimeEvidence = runtimeHits.Where(hit => path?.Contains(hit, StringComparer.Ordinal) == true)
|
||||
.ToList();
|
||||
if (runtimeEvidence.Count > 0)
|
||||
{
|
||||
confidence = Math.Min(scoringOptions.MaxConfidence, confidence + scoringOptions.RuntimeBonus);
|
||||
}
|
||||
var (bucket, weight, confidence) = ComputeScores(
|
||||
reachable,
|
||||
entryPoints,
|
||||
target,
|
||||
path,
|
||||
runtimeEvidence.Count);
|
||||
|
||||
confidence = Math.Clamp(confidence, scoringOptions.MinConfidence, scoringOptions.MaxConfidence);
|
||||
var score = confidence * weight;
|
||||
|
||||
states.Add(new ReachabilityStateDocument
|
||||
{
|
||||
Target = target,
|
||||
Reachable = reachable,
|
||||
Confidence = confidence,
|
||||
Bucket = bucket,
|
||||
Weight = weight,
|
||||
Score = score,
|
||||
Path = path ?? new List<string>(),
|
||||
Evidence = new ReachabilityEvidenceDocument
|
||||
{
|
||||
@@ -119,6 +125,14 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
});
|
||||
}
|
||||
|
||||
var baseScore = states.Count > 0 ? states.Average(s => s.Score) : 0;
|
||||
var unknownsCount = await unknownsRepository.CountBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
var pressure = states.Count + unknownsCount == 0
|
||||
? 0
|
||||
: Math.Min(1.0, Math.Max(0.0, unknownsCount / (double)(states.Count + unknownsCount)));
|
||||
var pressurePenalty = Math.Min(scoringOptions.UnknownsPenaltyCeiling, pressure);
|
||||
var finalScore = baseScore * (1 - pressurePenalty);
|
||||
|
||||
var document = new ReachabilityFactDocument
|
||||
{
|
||||
CallgraphId = request.CallgraphId,
|
||||
@@ -126,6 +140,9 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
EntryPoints = entryPoints,
|
||||
States = states,
|
||||
Metadata = request.Metadata,
|
||||
Score = finalScore,
|
||||
UnknownsCount = unknownsCount,
|
||||
UnknownsPressure = pressure,
|
||||
ComputedAt = timeProvider.GetUtcNow(),
|
||||
SubjectKey = subjectKey,
|
||||
RuntimeFacts = existingFact?.RuntimeFacts
|
||||
@@ -278,6 +295,51 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
return path;
|
||||
}
|
||||
|
||||
private (string bucket, double weight, double confidence) ComputeScores(
|
||||
bool reachable,
|
||||
List<string> entryPoints,
|
||||
string target,
|
||||
List<string>? path,
|
||||
int runtimeEvidenceCount)
|
||||
{
|
||||
var bucket = "unknown";
|
||||
if (!reachable)
|
||||
{
|
||||
bucket = "unreachable";
|
||||
}
|
||||
else if (entryPoints.Contains(target, StringComparer.Ordinal))
|
||||
{
|
||||
bucket = "entrypoint";
|
||||
}
|
||||
else if (runtimeEvidenceCount > 0)
|
||||
{
|
||||
bucket = "runtime";
|
||||
}
|
||||
else if (path is not null && path.Count <= 2)
|
||||
{
|
||||
bucket = "direct";
|
||||
}
|
||||
else
|
||||
{
|
||||
bucket = "unknown";
|
||||
}
|
||||
|
||||
var weight = scoringOptions.ReachabilityBuckets.TryGetValue(bucket, out var w)
|
||||
? w
|
||||
: scoringOptions.ReachabilityBuckets.TryGetValue("unknown", out var unknown)
|
||||
? unknown
|
||||
: 1.0;
|
||||
|
||||
var confidence = reachable ? scoringOptions.ReachableConfidence : scoringOptions.UnreachableConfidence;
|
||||
if (runtimeEvidenceCount > 0 && reachable)
|
||||
{
|
||||
confidence = Math.Min(scoringOptions.MaxConfidence, confidence + scoringOptions.RuntimeBonus);
|
||||
}
|
||||
|
||||
confidence = Math.Clamp(confidence, scoringOptions.MinConfidence, scoringOptions.MaxConfidence);
|
||||
return (bucket, weight, confidence);
|
||||
}
|
||||
|
||||
private sealed record ReachabilityGraph(
|
||||
HashSet<string> Nodes,
|
||||
Dictionary<string, HashSet<string>> Adjacency,
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Services.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Writes reachability union bundles (runtime + static) into the CAS layout: reachability_graphs/<analysisId>/
|
||||
/// Validates meta.json hashes before persisting.
|
||||
/// </summary>
|
||||
public sealed class ReachabilityUnionIngestionService : IReachabilityUnionIngestionService
|
||||
{
|
||||
private static readonly string[] RequiredFiles = { "nodes.ndjson", "edges.ndjson", "meta.json" };
|
||||
|
||||
private readonly ILogger<ReachabilityUnionIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
|
||||
public ReachabilityUnionIngestionService(
|
||||
ILogger<ReachabilityUnionIngestionService> logger,
|
||||
IOptions<SignalsOptions> options)
|
||||
{
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
public async Task<ReachabilityUnionIngestResponse> IngestAsync(string analysisId, Stream zipStream, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(analysisId);
|
||||
ArgumentNullException.ThrowIfNull(zipStream);
|
||||
|
||||
var casRoot = Path.Combine(options.Storage.RootPath, "reachability_graphs", analysisId.Trim());
|
||||
if (Directory.Exists(casRoot))
|
||||
{
|
||||
Directory.Delete(casRoot, recursive: true);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(casRoot);
|
||||
|
||||
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read, leaveOpen: true);
|
||||
|
||||
var entries = archive.Entries.ToDictionary(e => e.FullName, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var required in RequiredFiles)
|
||||
{
|
||||
if (!entries.ContainsKey(required))
|
||||
{
|
||||
throw new InvalidOperationException($"Union bundle missing required file: {required}");
|
||||
}
|
||||
}
|
||||
|
||||
var metaEntry = entries["meta.json"];
|
||||
using var metaStream = metaEntry.Open();
|
||||
using var metaDoc = await JsonDocument.ParseAsync(metaStream, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var metaRoot = metaDoc.RootElement;
|
||||
|
||||
var filesElement = metaRoot.TryGetProperty("files", out var f) && f.ValueKind == JsonValueKind.Array
|
||||
? f
|
||||
: throw new InvalidOperationException("meta.json is missing required 'files' array");
|
||||
|
||||
var recorded = filesElement.EnumerateArray()
|
||||
.Select(el => new
|
||||
{
|
||||
Path = el.GetProperty("path").GetString() ?? string.Empty,
|
||||
Sha = el.GetProperty("sha256").GetString() ?? string.Empty,
|
||||
Records = el.TryGetProperty("records", out var r) && r.ValueKind == JsonValueKind.Number ? r.GetInt32() : (int?)null
|
||||
})
|
||||
.ToList();
|
||||
|
||||
var filesForResponse = new List<ReachabilityUnionFile>();
|
||||
|
||||
foreach (var file in recorded)
|
||||
{
|
||||
if (!entries.TryGetValue(file.Path, out var zipEntry))
|
||||
{
|
||||
throw new InvalidOperationException($"meta.json references missing file '{file.Path}'.");
|
||||
}
|
||||
|
||||
var destPath = Path.Combine(casRoot, file.Path.Replace('/', Path.DirectorySeparatorChar));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destPath)!);
|
||||
|
||||
using (var entryStream = zipEntry.Open())
|
||||
using (var dest = File.Create(destPath))
|
||||
{
|
||||
await entryStream.CopyToAsync(dest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var actualSha = ComputeSha256Hex(destPath);
|
||||
if (!string.Equals(actualSha, file.Sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"SHA mismatch for {file.Path}: expected {file.Sha}, actual {actualSha}.");
|
||||
}
|
||||
|
||||
filesForResponse.Add(new ReachabilityUnionFile(file.Path, actualSha, file.Records));
|
||||
}
|
||||
|
||||
logger.LogInformation("Ingested reachability union bundle {AnalysisId} with {FileCount} files.", analysisId, filesForResponse.Count);
|
||||
|
||||
return new ReachabilityUnionIngestResponse(analysisId, $"cas://reachability_graphs/{analysisId}", filesForResponse);
|
||||
}
|
||||
|
||||
private static string ComputeSha256Hex(string path)
|
||||
{
|
||||
using var stream = File.OpenRead(path);
|
||||
var buffer = new byte[8192];
|
||||
using var sha = SHA256.Create();
|
||||
int read;
|
||||
while ((read = stream.Read(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
sha.TransformBlock(buffer, 0, read, null, 0);
|
||||
}
|
||||
|
||||
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
|
||||
return Convert.ToHexString(sha.Hash!).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
internal sealed class UnknownsIngestionService : IUnknownsIngestionService
|
||||
{
|
||||
private readonly IUnknownsRepository repository;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly ILogger<UnknownsIngestionService> logger;
|
||||
|
||||
public UnknownsIngestionService(IUnknownsRepository repository, TimeProvider timeProvider, ILogger<UnknownsIngestionService> logger)
|
||||
{
|
||||
this.repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<UnknownsIngestResponse> IngestAsync(UnknownsIngestRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (request.Subject is null)
|
||||
{
|
||||
throw new UnknownsValidationException("Subject is required.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.CallgraphId))
|
||||
{
|
||||
throw new UnknownsValidationException("callgraphId is required.");
|
||||
}
|
||||
|
||||
if (request.Unknowns is null || request.Unknowns.Count == 0)
|
||||
{
|
||||
throw new UnknownsValidationException("Unknowns list must not be empty.");
|
||||
}
|
||||
|
||||
var subjectKey = request.Subject.ToSubjectKey();
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
throw new UnknownsValidationException("Subject must include scanId, imageDigest, or component/version.");
|
||||
}
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var normalized = new List<UnknownSymbolDocument>();
|
||||
|
||||
foreach (var entry in request.Unknowns)
|
||||
{
|
||||
if (entry is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var hasContent = !(string.IsNullOrWhiteSpace(entry.SymbolId)
|
||||
&& string.IsNullOrWhiteSpace(entry.CodeId)
|
||||
&& string.IsNullOrWhiteSpace(entry.Purl)
|
||||
&& string.IsNullOrWhiteSpace(entry.EdgeFrom)
|
||||
&& string.IsNullOrWhiteSpace(entry.EdgeTo));
|
||||
|
||||
if (!hasContent)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
normalized.Add(new UnknownSymbolDocument
|
||||
{
|
||||
SubjectKey = subjectKey,
|
||||
CallgraphId = request.CallgraphId,
|
||||
SymbolId = entry.SymbolId?.Trim(),
|
||||
CodeId = entry.CodeId?.Trim(),
|
||||
Purl = entry.Purl?.Trim(),
|
||||
EdgeFrom = entry.EdgeFrom?.Trim(),
|
||||
EdgeTo = entry.EdgeTo?.Trim(),
|
||||
Reason = entry.Reason?.Trim(),
|
||||
CreatedAt = now
|
||||
});
|
||||
}
|
||||
|
||||
if (normalized.Count == 0)
|
||||
{
|
||||
throw new UnknownsValidationException("Unknown entries must include at least one symbolId, codeId, purl, or edge.");
|
||||
}
|
||||
|
||||
await repository.UpsertAsync(subjectKey, normalized, cancellationToken).ConfigureAwait(false);
|
||||
logger.LogInformation("Stored {Count} unknown symbols for subject {SubjectKey}", normalized.Count, subjectKey);
|
||||
|
||||
return new UnknownsIngestResponse
|
||||
{
|
||||
SubjectKey = subjectKey,
|
||||
UnknownsCount = normalized.Count
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
public sealed class UnknownsValidationException : Exception
|
||||
{
|
||||
public UnknownsValidationException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Services;
|
||||
using Xunit;
|
||||
|
||||
@@ -14,7 +15,7 @@ public class InMemoryEventsPublisherTests
|
||||
public async Task PublishFactUpdatedAsync_EmitsStructuredEvent()
|
||||
{
|
||||
var logger = new TestLogger<InMemoryEventsPublisher>();
|
||||
var publisher = new InMemoryEventsPublisher(logger);
|
||||
var publisher = new InMemoryEventsPublisher(logger, new SignalsOptions());
|
||||
|
||||
var fact = new ReachabilityFactDocument
|
||||
{
|
||||
@@ -23,8 +24,8 @@ public class InMemoryEventsPublisherTests
|
||||
ComputedAt = System.DateTimeOffset.Parse("2025-11-18T12:00:00Z"),
|
||||
States = new List<ReachabilityStateDocument>
|
||||
{
|
||||
new() { Target = "pkg:pypi/django", Reachable = true, Confidence = 0.9 },
|
||||
new() { Target = "pkg:pypi/requests", Reachable = false, Confidence = 0.2 }
|
||||
new() { Target = "pkg:pypi/django", Reachable = true, Confidence = 0.9, Bucket = "runtime", Weight = 0.45 },
|
||||
new() { Target = "pkg:pypi/requests", Reachable = false, Confidence = 0.2, Bucket = "runtime", Weight = 0.45 }
|
||||
},
|
||||
RuntimeFacts = new List<RuntimeFactDocument>
|
||||
{
|
||||
@@ -40,13 +41,20 @@ public class InMemoryEventsPublisherTests
|
||||
Assert.Contains("\"reachableCount\":1", logger.LastMessage);
|
||||
Assert.Contains("\"unreachableCount\":1", logger.LastMessage);
|
||||
Assert.Contains("\"runtimeFactsCount\":1", logger.LastMessage);
|
||||
Assert.Contains("\"bucket\":\"runtime\"", logger.LastMessage);
|
||||
Assert.Contains("\"weight\":0.45", logger.LastMessage);
|
||||
Assert.Contains("\"factScore\":", logger.LastMessage);
|
||||
Assert.Contains("\"unknownsCount\":0", logger.LastMessage);
|
||||
Assert.Contains("\"unknownsPressure\":0", logger.LastMessage);
|
||||
Assert.Contains("\"stateCount\":2", logger.LastMessage);
|
||||
Assert.Contains("\"targets\":[\"pkg:pypi/django\",\"pkg:pypi/requests\"]", logger.LastMessage);
|
||||
}
|
||||
|
||||
private sealed class TestLogger<T> : ILogger<T>
|
||||
{
|
||||
public string LastMessage { get; private set; } = string.Empty;
|
||||
|
||||
public IDisposable BeginScope<TState>(TState state) => NullScope.Instance;
|
||||
public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullScope.Instance;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ public class ReachabilityScoringServiceTests
|
||||
|
||||
var cache = new InMemoryReachabilityCache();
|
||||
var eventsPublisher = new RecordingEventsPublisher();
|
||||
var unknowns = new InMemoryUnknownsRepository();
|
||||
|
||||
var service = new ReachabilityScoringService(
|
||||
callgraphRepository,
|
||||
@@ -52,6 +53,7 @@ public class ReachabilityScoringServiceTests
|
||||
TimeProvider.System,
|
||||
Options.Create(options),
|
||||
cache,
|
||||
unknowns,
|
||||
eventsPublisher,
|
||||
NullLogger<ReachabilityScoringService>.Instance);
|
||||
|
||||
@@ -73,8 +75,13 @@ public class ReachabilityScoringServiceTests
|
||||
Assert.Equal("target", state.Target);
|
||||
Assert.Equal(new[] { "main", "svc", "target" }, state.Path);
|
||||
Assert.Equal(0.9, state.Confidence, 2); // 0.8 + 0.1 runtime bonus
|
||||
Assert.Equal("runtime", state.Bucket);
|
||||
Assert.Equal(0.45, state.Weight, 2);
|
||||
Assert.Equal(0.405, state.Score, 3);
|
||||
Assert.Contains("svc", state.Evidence.RuntimeHits);
|
||||
Assert.Contains("target", state.Evidence.RuntimeHits);
|
||||
|
||||
Assert.Equal(0.405, fact.Score, 3);
|
||||
}
|
||||
|
||||
private sealed class InMemoryCallgraphRepository : ICallgraphRepository
|
||||
@@ -147,4 +154,26 @@ public class ReachabilityScoringServiceTests
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryUnknownsRepository : IUnknownsRepository
|
||||
{
|
||||
public List<UnknownSymbolDocument> Stored { get; } = new();
|
||||
|
||||
public Task UpsertAsync(string subjectKey, IEnumerable<UnknownSymbolDocument> items, CancellationToken cancellationToken)
|
||||
{
|
||||
Stored.Clear();
|
||||
Stored.AddRange(items);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<UnknownSymbolDocument>> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult((IReadOnlyList<UnknownSymbolDocument>)Stored.ToList());
|
||||
}
|
||||
|
||||
public Task<int> CountBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(Stored.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Tests;
|
||||
|
||||
public class ReachabilityUnionIngestionServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task IngestAsync_ValidBundle_WritesFilesAndValidatesHashes()
|
||||
{
|
||||
// Arrange
|
||||
var tempRoot = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "signals-union-test-" + Guid.NewGuid().ToString("N")));
|
||||
var signalsOptions = new SignalsOptions();
|
||||
signalsOptions.Storage.RootPath = tempRoot.FullName;
|
||||
signalsOptions.Mongo.ConnectionString = "mongodb://localhost";
|
||||
signalsOptions.Mongo.Database = "stub";
|
||||
|
||||
var options = Microsoft.Extensions.Options.Options.Create(signalsOptions);
|
||||
|
||||
using var bundleStream = BuildSampleUnionZip();
|
||||
var service = new ReachabilityUnionIngestionService(NullLogger<ReachabilityUnionIngestionService>.Instance, options);
|
||||
|
||||
// Act
|
||||
var response = await service.IngestAsync("analysis-1", bundleStream, default);
|
||||
|
||||
// Assert
|
||||
Assert.Equal("analysis-1", response.AnalysisId);
|
||||
Assert.Contains(response.Files, f => f.Path == "nodes.ndjson");
|
||||
var metaPath = Path.Combine(tempRoot.FullName, "reachability_graphs", "analysis-1", "meta.json");
|
||||
Assert.True(File.Exists(metaPath));
|
||||
|
||||
// Cleanup
|
||||
tempRoot.Delete(true);
|
||||
}
|
||||
|
||||
private static MemoryStream BuildSampleUnionZip()
|
||||
{
|
||||
var ms = new MemoryStream();
|
||||
using var archive = new ZipArchive(ms, ZipArchiveMode.Create, leaveOpen: true);
|
||||
|
||||
var nodes = archive.CreateEntry("nodes.ndjson");
|
||||
using (var writer = new StreamWriter(nodes.Open()))
|
||||
{
|
||||
writer.WriteLine("{\"symbol_id\":\"sym:dotnet:abc\",\"lang\":\"dotnet\",\"kind\":\"function\",\"display\":\"abc\"}");
|
||||
}
|
||||
|
||||
var edges = archive.CreateEntry("edges.ndjson");
|
||||
using (var writer = new StreamWriter(edges.Open()))
|
||||
{
|
||||
writer.WriteLine("{\"from\":\"sym:dotnet:abc\",\"to\":\"sym:dotnet:def\",\"edge_type\":\"call\",\"source\":{\"origin\":\"static\",\"provenance\":\"il\"}}");
|
||||
}
|
||||
|
||||
// facts_runtime optional left out
|
||||
|
||||
var meta = archive.CreateEntry("meta.json");
|
||||
using (var writer = new StreamWriter(meta.Open()))
|
||||
{
|
||||
var files = new[]
|
||||
{
|
||||
new { path = "nodes.ndjson", sha256 = ComputeSha("{\"symbol_id\":\"sym:dotnet:abc\",\"lang\":\"dotnet\",\"kind\":\"function\",\"display\":\"abc\"}\n"), records = 1 },
|
||||
new { path = "edges.ndjson", sha256 = ComputeSha("{\"from\":\"sym:dotnet:abc\",\"to\":\"sym:dotnet:def\",\"edge_type\":\"call\",\"source\":{\"origin\":\"static\",\"provenance\":\"il\"}}\n"), records = 1 }
|
||||
};
|
||||
var metaObj = new
|
||||
{
|
||||
schema = "reachability-union@0.1",
|
||||
generated_at = "2025-11-23T00:00:00Z",
|
||||
produced_by = new { tool = "test", version = "0.0.1" },
|
||||
files
|
||||
};
|
||||
writer.Write(JsonSerializer.Serialize(metaObj));
|
||||
}
|
||||
|
||||
ms.Position = 0;
|
||||
return ms;
|
||||
}
|
||||
|
||||
private static string ComputeSha(string content)
|
||||
{
|
||||
using var sha = System.Security.Cryptography.SHA256.Create();
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(content);
|
||||
return Convert.ToHexString(sha.ComputeHash(bytes)).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Tests;
|
||||
|
||||
public class UnknownsIngestionServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task IngestAsync_StoresNormalizedUnknowns()
|
||||
{
|
||||
var repo = new InMemoryUnknownsRepository();
|
||||
var service = new UnknownsIngestionService(repo, TimeProvider.System, NullLogger<UnknownsIngestionService>.Instance);
|
||||
|
||||
var request = new UnknownsIngestRequest
|
||||
{
|
||||
Subject = new ReachabilitySubject { Component = "demo", Version = "1.0.0" },
|
||||
CallgraphId = "cg-1",
|
||||
Unknowns = new List<UnknownSymbolEntry>
|
||||
{
|
||||
new()
|
||||
{
|
||||
SymbolId = "symA",
|
||||
Purl = "pkg:pypi/foo",
|
||||
Reason = "missing-edge"
|
||||
},
|
||||
new() // empty entry should be ignored
|
||||
}
|
||||
};
|
||||
|
||||
var response = await service.IngestAsync(request, CancellationToken.None);
|
||||
|
||||
Assert.Equal("demo|1.0.0", response.SubjectKey);
|
||||
Assert.Equal(1, response.UnknownsCount);
|
||||
Assert.Single(repo.Stored);
|
||||
Assert.Equal("symA", repo.Stored[0].SymbolId);
|
||||
Assert.Equal("pkg:pypi/foo", repo.Stored[0].Purl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_ThrowsWhenEmpty()
|
||||
{
|
||||
var repo = new InMemoryUnknownsRepository();
|
||||
var service = new UnknownsIngestionService(repo, TimeProvider.System, NullLogger<UnknownsIngestionService>.Instance);
|
||||
|
||||
var request = new UnknownsIngestRequest
|
||||
{
|
||||
Subject = new ReachabilitySubject { Component = "demo", Version = "1.0.0" },
|
||||
CallgraphId = "cg-1",
|
||||
Unknowns = new List<UnknownSymbolEntry>()
|
||||
};
|
||||
|
||||
await Assert.ThrowsAsync<UnknownsValidationException>(() => service.IngestAsync(request, CancellationToken.None));
|
||||
}
|
||||
|
||||
private sealed class InMemoryUnknownsRepository : IUnknownsRepository
|
||||
{
|
||||
public List<UnknownSymbolDocument> Stored { get; } = new();
|
||||
|
||||
public Task UpsertAsync(string subjectKey, IEnumerable<UnknownSymbolDocument> items, CancellationToken cancellationToken)
|
||||
{
|
||||
Stored.Clear();
|
||||
Stored.AddRange(items);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<UnknownSymbolDocument>> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult((IReadOnlyList<UnknownSymbolDocument>)Stored);
|
||||
}
|
||||
|
||||
public Task<int> CountBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(Stored.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user