up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,150 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MicrosoftOptions = Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Storage.Postgres.Repositories;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Postgres.Tests;
|
||||
|
||||
[Collection(SignalsPostgresCollection.Name)]
|
||||
public sealed class PostgresCallgraphRepositoryTests : IAsyncLifetime
|
||||
{
|
||||
private readonly SignalsPostgresFixture _fixture;
|
||||
private readonly PostgresCallgraphRepository _repository;
|
||||
|
||||
public PostgresCallgraphRepositoryTests(SignalsPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
|
||||
var options = fixture.Fixture.CreateOptions();
|
||||
options.SchemaName = fixture.SchemaName;
|
||||
var dataSource = new SignalsDataSource(MicrosoftOptions.Options.Create(options), NullLogger<SignalsDataSource>.Instance);
|
||||
_repository = new PostgresCallgraphRepository(dataSource, NullLogger<PostgresCallgraphRepository>.Instance);
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.TruncateAllTablesAsync();
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertAndGetById_RoundTripsCallgraphDocument()
|
||||
{
|
||||
// Arrange
|
||||
var id = "callgraph-" + Guid.NewGuid().ToString("N");
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
Id = id,
|
||||
Language = "javascript",
|
||||
Component = "pkg:npm/lodash@4.17.21",
|
||||
Version = "4.17.21",
|
||||
IngestedAt = DateTimeOffset.UtcNow,
|
||||
GraphHash = "sha256:abc123",
|
||||
Nodes = new List<CallgraphNode>
|
||||
{
|
||||
new("fn1", "main", "function", "lodash", "index.js", 1),
|
||||
new("fn2", "helper", "function", "lodash", "utils.js", 10)
|
||||
},
|
||||
Edges = new List<CallgraphEdge>
|
||||
{
|
||||
new("fn1", "fn2", "call")
|
||||
},
|
||||
Metadata = new Dictionary<string, string?> { ["version"] = "1.0" }
|
||||
};
|
||||
|
||||
// Act
|
||||
await _repository.UpsertAsync(document, CancellationToken.None);
|
||||
var fetched = await _repository.GetByIdAsync(id, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.Id.Should().Be(id);
|
||||
fetched.Language.Should().Be("javascript");
|
||||
fetched.Component.Should().Be("pkg:npm/lodash@4.17.21");
|
||||
fetched.Nodes.Should().HaveCount(2);
|
||||
fetched.Edges.Should().HaveCount(1);
|
||||
fetched.Metadata.Should().ContainKey("version");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertAsync_UpdatesExistingDocument()
|
||||
{
|
||||
// Arrange
|
||||
var id = "callgraph-" + Guid.NewGuid().ToString("N");
|
||||
var document1 = new CallgraphDocument
|
||||
{
|
||||
Id = id,
|
||||
Language = "javascript",
|
||||
Component = "pkg:npm/express@4.18.0",
|
||||
Version = "4.18.0",
|
||||
IngestedAt = DateTimeOffset.UtcNow,
|
||||
GraphHash = "hash1",
|
||||
Nodes = new List<CallgraphNode> { new("fn1", "old", "function", null, "a.js", 1) },
|
||||
Edges = new List<CallgraphEdge>()
|
||||
};
|
||||
|
||||
var document2 = new CallgraphDocument
|
||||
{
|
||||
Id = id,
|
||||
Language = "typescript",
|
||||
Component = "pkg:npm/express@4.19.0",
|
||||
Version = "4.19.0",
|
||||
IngestedAt = DateTimeOffset.UtcNow,
|
||||
GraphHash = "hash2",
|
||||
Nodes = new List<CallgraphNode>
|
||||
{
|
||||
new("fn1", "new1", "function", null, "b.js", 1),
|
||||
new("fn2", "new2", "function", null, "b.js", 5)
|
||||
},
|
||||
Edges = new List<CallgraphEdge>()
|
||||
};
|
||||
|
||||
// Act
|
||||
await _repository.UpsertAsync(document1, CancellationToken.None);
|
||||
await _repository.UpsertAsync(document2, CancellationToken.None);
|
||||
var fetched = await _repository.GetByIdAsync(id, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
fetched.Should().NotBeNull();
|
||||
fetched!.Language.Should().Be("typescript");
|
||||
fetched.Version.Should().Be("4.19.0");
|
||||
fetched.Nodes.Should().HaveCount(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByIdAsync_ReturnsNullForNonExistentId()
|
||||
{
|
||||
// Act
|
||||
var fetched = await _repository.GetByIdAsync("nonexistent-id", CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
fetched.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertAsync_GeneratesIdIfMissing()
|
||||
{
|
||||
// Arrange
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
Id = string.Empty, // empty ID should be replaced
|
||||
Language = "python",
|
||||
Component = "pkg:pypi/requests@2.28.0",
|
||||
Version = "2.28.0",
|
||||
IngestedAt = DateTimeOffset.UtcNow,
|
||||
GraphHash = "hash123",
|
||||
Nodes = new List<CallgraphNode>(),
|
||||
Edges = new List<CallgraphEdge>()
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await _repository.UpsertAsync(document, CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
result.Id.Should().NotBeNullOrWhiteSpace();
|
||||
result.Id.Should().HaveLength(32); // GUID without hyphens
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using System.Reflection;
|
||||
using StellaOps.Infrastructure.Postgres.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Postgres.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL integration test fixture for the Signals module.
|
||||
/// </summary>
|
||||
public sealed class SignalsPostgresFixture : PostgresIntegrationFixture, ICollectionFixture<SignalsPostgresFixture>
|
||||
{
|
||||
protected override Assembly? GetMigrationAssembly()
|
||||
=> typeof(SignalsDataSource).Assembly;
|
||||
|
||||
protected override string GetModuleName() => "Signals";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Collection definition for Signals PostgreSQL integration tests.
|
||||
/// Tests in this collection share a single PostgreSQL container instance.
|
||||
/// </summary>
|
||||
[CollectionDefinition(Name)]
|
||||
public sealed class SignalsPostgresCollection : ICollectionFixture<SignalsPostgresFixture>
|
||||
{
|
||||
public const string Name = "SignalsPostgres";
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
<?xml version="1.0" ?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Signals.Storage.Postgres\StellaOps.Signals.Storage.Postgres.csproj" />
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.Infrastructure.Postgres.Testing\StellaOps.Infrastructure.Postgres.Testing.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,128 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of <see cref="ICallgraphRepository"/>.
|
||||
/// </summary>
|
||||
public sealed class PostgresCallgraphRepository : RepositoryBase<SignalsDataSource>, ICallgraphRepository
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private bool _tableInitialized;
|
||||
|
||||
public PostgresCallgraphRepository(SignalsDataSource dataSource, ILogger<PostgresCallgraphRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task<CallgraphDocument> UpsertAsync(CallgraphDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(document.Id))
|
||||
{
|
||||
document.Id = Guid.NewGuid().ToString("N");
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
INSERT INTO signals.callgraphs (id, language, component, version, graph_hash, ingested_at, document_json)
|
||||
VALUES (@id, @language, @component, @version, @graph_hash, @ingested_at, @document_json)
|
||||
ON CONFLICT (id)
|
||||
DO UPDATE SET
|
||||
language = EXCLUDED.language,
|
||||
component = EXCLUDED.component,
|
||||
version = EXCLUDED.version,
|
||||
graph_hash = EXCLUDED.graph_hash,
|
||||
ingested_at = EXCLUDED.ingested_at,
|
||||
document_json = EXCLUDED.document_json
|
||||
RETURNING id";
|
||||
|
||||
var documentJson = JsonSerializer.Serialize(document, JsonOptions);
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
|
||||
AddParameter(command, "@id", document.Id);
|
||||
AddParameter(command, "@language", document.Language ?? string.Empty);
|
||||
AddParameter(command, "@component", document.Component ?? string.Empty);
|
||||
AddParameter(command, "@version", document.Version ?? string.Empty);
|
||||
AddParameter(command, "@graph_hash", document.GraphHash ?? string.Empty);
|
||||
AddParameter(command, "@ingested_at", document.IngestedAt);
|
||||
AddJsonbParameter(command, "@document_json", documentJson);
|
||||
|
||||
await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public async Task<CallgraphDocument?> GetByIdAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT document_json
|
||||
FROM signals.callgraphs
|
||||
WHERE id = @id";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@id", id.Trim());
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var documentJson = reader.GetString(0);
|
||||
return JsonSerializer.Deserialize<CallgraphDocument>(documentJson, JsonOptions);
|
||||
}
|
||||
|
||||
private async Task EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tableInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string ddl = @"
|
||||
CREATE SCHEMA IF NOT EXISTS signals;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signals.callgraphs (
|
||||
id TEXT PRIMARY KEY,
|
||||
language TEXT NOT NULL,
|
||||
component TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
graph_hash TEXT NOT NULL,
|
||||
ingested_at TIMESTAMPTZ NOT NULL,
|
||||
document_json JSONB NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_callgraphs_component ON signals.callgraphs (component);
|
||||
CREATE INDEX IF NOT EXISTS idx_callgraphs_graph_hash ON signals.callgraphs (graph_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_callgraphs_ingested_at ON signals.callgraphs (ingested_at DESC);";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(ddl, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_tableInitialized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of <see cref="IReachabilityFactRepository"/>.
|
||||
/// </summary>
|
||||
public sealed class PostgresReachabilityFactRepository : RepositoryBase<SignalsDataSource>, IReachabilityFactRepository
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private bool _tableInitialized;
|
||||
|
||||
public PostgresReachabilityFactRepository(SignalsDataSource dataSource, ILogger<PostgresReachabilityFactRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task<ReachabilityFactDocument> UpsertAsync(ReachabilityFactDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(document.SubjectKey))
|
||||
{
|
||||
throw new ArgumentException("Subject key is required.", nameof(document));
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
INSERT INTO signals.reachability_facts (subject_key, id, callgraph_id, score, risk_score, computed_at, document_json)
|
||||
VALUES (@subject_key, @id, @callgraph_id, @score, @risk_score, @computed_at, @document_json)
|
||||
ON CONFLICT (subject_key)
|
||||
DO UPDATE SET
|
||||
id = EXCLUDED.id,
|
||||
callgraph_id = EXCLUDED.callgraph_id,
|
||||
score = EXCLUDED.score,
|
||||
risk_score = EXCLUDED.risk_score,
|
||||
computed_at = EXCLUDED.computed_at,
|
||||
document_json = EXCLUDED.document_json
|
||||
RETURNING subject_key";
|
||||
|
||||
var documentJson = JsonSerializer.Serialize(document, JsonOptions);
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
|
||||
AddParameter(command, "@subject_key", document.SubjectKey);
|
||||
AddParameter(command, "@id", document.Id);
|
||||
AddParameter(command, "@callgraph_id", document.CallgraphId ?? string.Empty);
|
||||
AddParameter(command, "@score", document.Score);
|
||||
AddParameter(command, "@risk_score", document.RiskScore);
|
||||
AddParameter(command, "@computed_at", document.ComputedAt);
|
||||
AddJsonbParameter(command, "@document_json", documentJson);
|
||||
|
||||
await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public async Task<ReachabilityFactDocument?> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
throw new ArgumentException("Subject key is required.", nameof(subjectKey));
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT document_json
|
||||
FROM signals.reachability_facts
|
||||
WHERE subject_key = @subject_key";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@subject_key", subjectKey.Trim());
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var documentJson = reader.GetString(0);
|
||||
return JsonSerializer.Deserialize<ReachabilityFactDocument>(documentJson, JsonOptions);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<ReachabilityFactDocument>> GetExpiredAsync(DateTimeOffset cutoff, int limit, CancellationToken cancellationToken)
|
||||
{
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT document_json
|
||||
FROM signals.reachability_facts
|
||||
WHERE computed_at < @cutoff
|
||||
ORDER BY computed_at ASC
|
||||
LIMIT @limit";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@cutoff", cutoff);
|
||||
AddParameter(command, "@limit", limit);
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<ReachabilityFactDocument>();
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var documentJson = reader.GetString(0);
|
||||
var document = JsonSerializer.Deserialize<ReachabilityFactDocument>(documentJson, JsonOptions);
|
||||
if (document is not null)
|
||||
{
|
||||
results.Add(document);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
throw new ArgumentException("Subject key is required.", nameof(subjectKey));
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
DELETE FROM signals.reachability_facts
|
||||
WHERE subject_key = @subject_key";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@subject_key", subjectKey.Trim());
|
||||
|
||||
var rows = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
return rows > 0;
|
||||
}
|
||||
|
||||
public async Task<int> GetRuntimeFactsCountAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
throw new ArgumentException("Subject key is required.", nameof(subjectKey));
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT COALESCE(jsonb_array_length(document_json->'runtimeFacts'), 0)
|
||||
FROM signals.reachability_facts
|
||||
WHERE subject_key = @subject_key";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@subject_key", subjectKey.Trim());
|
||||
|
||||
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
return result is int count ? count : 0;
|
||||
}
|
||||
|
||||
public async Task TrimRuntimeFactsAsync(string subjectKey, int maxCount, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
throw new ArgumentException("Subject key is required.", nameof(subjectKey));
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Get the document, trim in memory, and update
|
||||
var document = await GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
if (document?.RuntimeFacts is not { Count: > 0 })
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (document.RuntimeFacts.Count <= maxCount)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var trimmed = document.RuntimeFacts
|
||||
.OrderByDescending(f => f.ObservedAt ?? DateTimeOffset.MinValue)
|
||||
.ThenByDescending(f => f.HitCount)
|
||||
.Take(maxCount)
|
||||
.ToList();
|
||||
|
||||
document.RuntimeFacts = trimmed;
|
||||
await UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tableInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string ddl = @"
|
||||
CREATE SCHEMA IF NOT EXISTS signals;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signals.reachability_facts (
|
||||
subject_key TEXT PRIMARY KEY,
|
||||
id TEXT NOT NULL,
|
||||
callgraph_id TEXT NOT NULL,
|
||||
score DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
risk_score DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
computed_at TIMESTAMPTZ NOT NULL,
|
||||
document_json JSONB NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_reachability_facts_callgraph_id ON signals.reachability_facts (callgraph_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_reachability_facts_computed_at ON signals.reachability_facts (computed_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_reachability_facts_score ON signals.reachability_facts (score DESC);";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(ddl, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_tableInitialized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Models.ReachabilityStore;
|
||||
using StellaOps.Signals.Persistence;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of <see cref="IReachabilityStoreRepository"/>.
|
||||
/// </summary>
|
||||
public sealed class PostgresReachabilityStoreRepository : RepositoryBase<SignalsDataSource>, IReachabilityStoreRepository
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false
|
||||
};
|
||||
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private bool _tableInitialized;
|
||||
|
||||
public PostgresReachabilityStoreRepository(
|
||||
SignalsDataSource dataSource,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<PostgresReachabilityStoreRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public async Task UpsertGraphAsync(
|
||||
string graphHash,
|
||||
IReadOnlyCollection<CallgraphNode> nodes,
|
||||
IReadOnlyCollection<CallgraphEdge> edges,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(graphHash);
|
||||
ArgumentNullException.ThrowIfNull(nodes);
|
||||
ArgumentNullException.ThrowIfNull(edges);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var normalizedGraphHash = graphHash.Trim();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
// Upsert func nodes
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
var symbolId = node.Id?.Trim() ?? string.Empty;
|
||||
var id = $"{normalizedGraphHash}|{symbolId}";
|
||||
|
||||
const string nodeSql = @"
|
||||
INSERT INTO signals.func_nodes (id, graph_hash, symbol_id, name, kind, namespace, file, line, purl, symbol_digest, build_id, code_id, language, evidence, analyzer, ingested_at)
|
||||
VALUES (@id, @graph_hash, @symbol_id, @name, @kind, @namespace, @file, @line, @purl, @symbol_digest, @build_id, @code_id, @language, @evidence, @analyzer, @ingested_at)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
kind = EXCLUDED.kind,
|
||||
namespace = EXCLUDED.namespace,
|
||||
file = EXCLUDED.file,
|
||||
line = EXCLUDED.line,
|
||||
purl = EXCLUDED.purl,
|
||||
symbol_digest = EXCLUDED.symbol_digest,
|
||||
build_id = EXCLUDED.build_id,
|
||||
code_id = EXCLUDED.code_id,
|
||||
language = EXCLUDED.language,
|
||||
evidence = EXCLUDED.evidence,
|
||||
analyzer = EXCLUDED.analyzer,
|
||||
ingested_at = EXCLUDED.ingested_at";
|
||||
|
||||
await using var nodeCommand = CreateCommand(nodeSql, connection, transaction);
|
||||
AddParameter(nodeCommand, "@id", id);
|
||||
AddParameter(nodeCommand, "@graph_hash", normalizedGraphHash);
|
||||
AddParameter(nodeCommand, "@symbol_id", symbolId);
|
||||
AddParameter(nodeCommand, "@name", node.Name?.Trim() ?? string.Empty);
|
||||
AddParameter(nodeCommand, "@kind", node.Kind?.Trim() ?? string.Empty);
|
||||
AddParameter(nodeCommand, "@namespace", (object?)node.Namespace?.Trim() ?? DBNull.Value);
|
||||
AddParameter(nodeCommand, "@file", (object?)node.File?.Trim() ?? DBNull.Value);
|
||||
AddParameter(nodeCommand, "@line", (object?)node.Line ?? DBNull.Value);
|
||||
AddParameter(nodeCommand, "@purl", (object?)node.Purl?.Trim() ?? DBNull.Value);
|
||||
AddParameter(nodeCommand, "@symbol_digest", (object?)node.SymbolDigest?.Trim()?.ToLowerInvariant() ?? DBNull.Value);
|
||||
AddParameter(nodeCommand, "@build_id", (object?)node.BuildId?.Trim() ?? DBNull.Value);
|
||||
AddParameter(nodeCommand, "@code_id", (object?)node.CodeId?.Trim() ?? DBNull.Value);
|
||||
AddParameter(nodeCommand, "@language", (object?)node.Language?.Trim() ?? DBNull.Value);
|
||||
AddJsonbParameter(nodeCommand, "@evidence", node.Evidence is null ? null : JsonSerializer.Serialize(node.Evidence, JsonOptions));
|
||||
AddJsonbParameter(nodeCommand, "@analyzer", node.Analyzer is null ? null : JsonSerializer.Serialize(node.Analyzer, JsonOptions));
|
||||
AddParameter(nodeCommand, "@ingested_at", now);
|
||||
|
||||
await nodeCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Upsert call edges
|
||||
foreach (var edge in edges)
|
||||
{
|
||||
var sourceId = edge.SourceId?.Trim() ?? string.Empty;
|
||||
var targetId = edge.TargetId?.Trim() ?? string.Empty;
|
||||
var type = edge.Type?.Trim() ?? string.Empty;
|
||||
var id = $"{normalizedGraphHash}|{sourceId}->{targetId}|{type}";
|
||||
|
||||
const string edgeSql = @"
|
||||
INSERT INTO signals.call_edges (id, graph_hash, source_id, target_id, type, purl, symbol_digest, candidates, confidence, evidence, ingested_at)
|
||||
VALUES (@id, @graph_hash, @source_id, @target_id, @type, @purl, @symbol_digest, @candidates, @confidence, @evidence, @ingested_at)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
purl = EXCLUDED.purl,
|
||||
symbol_digest = EXCLUDED.symbol_digest,
|
||||
candidates = EXCLUDED.candidates,
|
||||
confidence = EXCLUDED.confidence,
|
||||
evidence = EXCLUDED.evidence,
|
||||
ingested_at = EXCLUDED.ingested_at";
|
||||
|
||||
await using var edgeCommand = CreateCommand(edgeSql, connection, transaction);
|
||||
AddParameter(edgeCommand, "@id", id);
|
||||
AddParameter(edgeCommand, "@graph_hash", normalizedGraphHash);
|
||||
AddParameter(edgeCommand, "@source_id", sourceId);
|
||||
AddParameter(edgeCommand, "@target_id", targetId);
|
||||
AddParameter(edgeCommand, "@type", type);
|
||||
AddParameter(edgeCommand, "@purl", (object?)edge.Purl?.Trim() ?? DBNull.Value);
|
||||
AddParameter(edgeCommand, "@symbol_digest", (object?)edge.SymbolDigest?.Trim()?.ToLowerInvariant() ?? DBNull.Value);
|
||||
AddJsonbParameter(edgeCommand, "@candidates", edge.Candidates is null ? null : JsonSerializer.Serialize(edge.Candidates, JsonOptions));
|
||||
AddParameter(edgeCommand, "@confidence", (object?)edge.Confidence ?? DBNull.Value);
|
||||
AddJsonbParameter(edgeCommand, "@evidence", edge.Evidence is null ? null : JsonSerializer.Serialize(edge.Evidence, JsonOptions));
|
||||
AddParameter(edgeCommand, "@ingested_at", now);
|
||||
|
||||
await edgeCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<FuncNodeDocument>> GetFuncNodesByGraphAsync(string graphHash, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(graphHash))
|
||||
{
|
||||
return Array.Empty<FuncNodeDocument>();
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT id, graph_hash, symbol_id, name, kind, namespace, file, line, purl, symbol_digest, build_id, code_id, language, evidence, analyzer, ingested_at
|
||||
FROM signals.func_nodes
|
||||
WHERE graph_hash = @graph_hash
|
||||
ORDER BY symbol_id, purl, symbol_digest";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@graph_hash", graphHash.Trim());
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<FuncNodeDocument>();
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(MapFuncNode(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CallEdgeDocument>> GetCallEdgesByGraphAsync(string graphHash, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(graphHash))
|
||||
{
|
||||
return Array.Empty<CallEdgeDocument>();
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT id, graph_hash, source_id, target_id, type, purl, symbol_digest, candidates, confidence, evidence, ingested_at
|
||||
FROM signals.call_edges
|
||||
WHERE graph_hash = @graph_hash
|
||||
ORDER BY source_id, target_id, type";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@graph_hash", graphHash.Trim());
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<CallEdgeDocument>();
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(MapCallEdge(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
public async Task UpsertCveFuncHitsAsync(IReadOnlyCollection<CveFuncHitDocument> hits, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(hits);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var hit in hits.Where(h => h is not null))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(hit.SubjectKey) || string.IsNullOrWhiteSpace(hit.CveId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = $"{hit.SubjectKey.Trim()}|{hit.CveId.Trim().ToUpperInvariant()}|{hit.Purl?.Trim() ?? string.Empty}|{hit.SymbolDigest?.Trim()?.ToLowerInvariant() ?? string.Empty}";
|
||||
|
||||
const string sql = @"
|
||||
INSERT INTO signals.cve_func_hits (id, subject_key, cve_id, graph_hash, purl, symbol_digest, reachable, confidence, lattice_state, evidence_uris, computed_at)
|
||||
VALUES (@id, @subject_key, @cve_id, @graph_hash, @purl, @symbol_digest, @reachable, @confidence, @lattice_state, @evidence_uris, @computed_at)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
graph_hash = EXCLUDED.graph_hash,
|
||||
reachable = EXCLUDED.reachable,
|
||||
confidence = EXCLUDED.confidence,
|
||||
lattice_state = EXCLUDED.lattice_state,
|
||||
evidence_uris = EXCLUDED.evidence_uris,
|
||||
computed_at = EXCLUDED.computed_at";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@id", id);
|
||||
AddParameter(command, "@subject_key", hit.SubjectKey.Trim());
|
||||
AddParameter(command, "@cve_id", hit.CveId.Trim().ToUpperInvariant());
|
||||
AddParameter(command, "@graph_hash", hit.GraphHash ?? string.Empty);
|
||||
AddParameter(command, "@purl", (object?)hit.Purl?.Trim() ?? DBNull.Value);
|
||||
AddParameter(command, "@symbol_digest", (object?)hit.SymbolDigest?.Trim()?.ToLowerInvariant() ?? DBNull.Value);
|
||||
AddParameter(command, "@reachable", hit.Reachable);
|
||||
AddParameter(command, "@confidence", (object?)hit.Confidence ?? DBNull.Value);
|
||||
AddParameter(command, "@lattice_state", (object?)hit.LatticeState ?? DBNull.Value);
|
||||
AddJsonbParameter(command, "@evidence_uris", hit.EvidenceUris is null ? null : JsonSerializer.Serialize(hit.EvidenceUris, JsonOptions));
|
||||
AddParameter(command, "@computed_at", hit.ComputedAt);
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CveFuncHitDocument>> GetCveFuncHitsBySubjectAsync(
|
||||
string subjectKey,
|
||||
string cveId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(subjectKey) || string.IsNullOrWhiteSpace(cveId))
|
||||
{
|
||||
return Array.Empty<CveFuncHitDocument>();
|
||||
}
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT id, subject_key, cve_id, graph_hash, purl, symbol_digest, reachable, confidence, lattice_state, evidence_uris, computed_at
|
||||
FROM signals.cve_func_hits
|
||||
WHERE subject_key = @subject_key AND UPPER(cve_id) = UPPER(@cve_id)
|
||||
ORDER BY cve_id, purl, symbol_digest";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@subject_key", subjectKey.Trim());
|
||||
AddParameter(command, "@cve_id", cveId.Trim());
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<CveFuncHitDocument>();
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(MapCveFuncHit(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static FuncNodeDocument MapFuncNode(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(0),
|
||||
GraphHash = reader.GetString(1),
|
||||
SymbolId = reader.GetString(2),
|
||||
Name = reader.GetString(3),
|
||||
Kind = reader.GetString(4),
|
||||
Namespace = reader.IsDBNull(5) ? null : reader.GetString(5),
|
||||
File = reader.IsDBNull(6) ? null : reader.GetString(6),
|
||||
Line = reader.IsDBNull(7) ? null : reader.GetInt32(7),
|
||||
Purl = reader.IsDBNull(8) ? null : reader.GetString(8),
|
||||
SymbolDigest = reader.IsDBNull(9) ? null : reader.GetString(9),
|
||||
BuildId = reader.IsDBNull(10) ? null : reader.GetString(10),
|
||||
CodeId = reader.IsDBNull(11) ? null : reader.GetString(11),
|
||||
Language = reader.IsDBNull(12) ? null : reader.GetString(12),
|
||||
Evidence = reader.IsDBNull(13) ? null : JsonSerializer.Deserialize<List<string>>(reader.GetString(13), JsonOptions),
|
||||
Analyzer = reader.IsDBNull(14) ? null : JsonSerializer.Deserialize<Dictionary<string, string?>>(reader.GetString(14), JsonOptions),
|
||||
IngestedAt = reader.GetFieldValue<DateTimeOffset>(15)
|
||||
};
|
||||
|
||||
private static CallEdgeDocument MapCallEdge(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(0),
|
||||
GraphHash = reader.GetString(1),
|
||||
SourceId = reader.GetString(2),
|
||||
TargetId = reader.GetString(3),
|
||||
Type = reader.GetString(4),
|
||||
Purl = reader.IsDBNull(5) ? null : reader.GetString(5),
|
||||
SymbolDigest = reader.IsDBNull(6) ? null : reader.GetString(6),
|
||||
Candidates = reader.IsDBNull(7) ? null : JsonSerializer.Deserialize<List<string>>(reader.GetString(7), JsonOptions),
|
||||
Confidence = reader.IsDBNull(8) ? null : reader.GetDouble(8),
|
||||
Evidence = reader.IsDBNull(9) ? null : JsonSerializer.Deserialize<List<string>>(reader.GetString(9), JsonOptions),
|
||||
IngestedAt = reader.GetFieldValue<DateTimeOffset>(10)
|
||||
};
|
||||
|
||||
private static CveFuncHitDocument MapCveFuncHit(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(0),
|
||||
SubjectKey = reader.GetString(1),
|
||||
CveId = reader.GetString(2),
|
||||
GraphHash = reader.GetString(3),
|
||||
Purl = reader.IsDBNull(4) ? null : reader.GetString(4),
|
||||
SymbolDigest = reader.IsDBNull(5) ? null : reader.GetString(5),
|
||||
Reachable = reader.GetBoolean(6),
|
||||
Confidence = reader.IsDBNull(7) ? null : reader.GetDouble(7),
|
||||
LatticeState = reader.IsDBNull(8) ? null : reader.GetString(8),
|
||||
EvidenceUris = reader.IsDBNull(9) ? null : JsonSerializer.Deserialize<List<string>>(reader.GetString(9), JsonOptions),
|
||||
ComputedAt = reader.GetFieldValue<DateTimeOffset>(10)
|
||||
};
|
||||
|
||||
private static NpgsqlCommand CreateCommand(string sql, NpgsqlConnection connection, NpgsqlTransaction transaction)
|
||||
{
|
||||
var command = new NpgsqlCommand(sql, connection, transaction);
|
||||
return command;
|
||||
}
|
||||
|
||||
private async Task EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tableInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string ddl = @"
|
||||
CREATE SCHEMA IF NOT EXISTS signals;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signals.func_nodes (
|
||||
id TEXT PRIMARY KEY,
|
||||
graph_hash TEXT NOT NULL,
|
||||
symbol_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
namespace TEXT,
|
||||
file TEXT,
|
||||
line INTEGER,
|
||||
purl TEXT,
|
||||
symbol_digest TEXT,
|
||||
build_id TEXT,
|
||||
code_id TEXT,
|
||||
language TEXT,
|
||||
evidence JSONB,
|
||||
analyzer JSONB,
|
||||
ingested_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_func_nodes_graph_hash ON signals.func_nodes (graph_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_func_nodes_symbol_digest ON signals.func_nodes (symbol_digest) WHERE symbol_digest IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_func_nodes_purl ON signals.func_nodes (purl) WHERE purl IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signals.call_edges (
|
||||
id TEXT PRIMARY KEY,
|
||||
graph_hash TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
target_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
purl TEXT,
|
||||
symbol_digest TEXT,
|
||||
candidates JSONB,
|
||||
confidence DOUBLE PRECISION,
|
||||
evidence JSONB,
|
||||
ingested_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_call_edges_graph_hash ON signals.call_edges (graph_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_call_edges_source_id ON signals.call_edges (source_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_call_edges_target_id ON signals.call_edges (target_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signals.cve_func_hits (
|
||||
id TEXT PRIMARY KEY,
|
||||
subject_key TEXT NOT NULL,
|
||||
cve_id TEXT NOT NULL,
|
||||
graph_hash TEXT NOT NULL,
|
||||
purl TEXT,
|
||||
symbol_digest TEXT,
|
||||
reachable BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
confidence DOUBLE PRECISION,
|
||||
lattice_state TEXT,
|
||||
evidence_uris JSONB,
|
||||
computed_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cve_func_hits_subject_key ON signals.cve_func_hits (subject_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_cve_func_hits_cve_id ON signals.cve_func_hits (UPPER(cve_id));
|
||||
CREATE INDEX IF NOT EXISTS idx_cve_func_hits_subject_cve ON signals.cve_func_hits (subject_key, UPPER(cve_id));";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(ddl, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_tableInitialized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Repositories;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Postgres.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL implementation of <see cref="IUnknownsRepository"/>.
|
||||
/// </summary>
|
||||
public sealed class PostgresUnknownsRepository : RepositoryBase<SignalsDataSource>, IUnknownsRepository
|
||||
{
|
||||
private bool _tableInitialized;
|
||||
|
||||
public PostgresUnknownsRepository(SignalsDataSource dataSource, ILogger<PostgresUnknownsRepository> logger)
|
||||
: base(dataSource, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public async Task UpsertAsync(string subjectKey, IEnumerable<UnknownSymbolDocument> items, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
|
||||
ArgumentNullException.ThrowIfNull(items);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var normalizedSubjectKey = subjectKey.Trim();
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
// Delete existing items for this subject
|
||||
const string deleteSql = "DELETE FROM signals.unknowns WHERE subject_key = @subject_key";
|
||||
await using (var deleteCommand = CreateCommand(deleteSql, connection, transaction))
|
||||
{
|
||||
AddParameter(deleteCommand, "@subject_key", normalizedSubjectKey);
|
||||
await deleteCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Insert new items
|
||||
const string insertSql = @"
|
||||
INSERT INTO signals.unknowns (id, subject_key, callgraph_id, symbol_id, code_id, purl, edge_from, edge_to, reason, created_at)
|
||||
VALUES (@id, @subject_key, @callgraph_id, @symbol_id, @code_id, @purl, @edge_from, @edge_to, @reason, @created_at)";
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (item is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var itemId = string.IsNullOrWhiteSpace(item.Id) ? Guid.NewGuid().ToString("N") : item.Id.Trim();
|
||||
|
||||
await using var insertCommand = CreateCommand(insertSql, connection, transaction);
|
||||
AddParameter(insertCommand, "@id", itemId);
|
||||
AddParameter(insertCommand, "@subject_key", normalizedSubjectKey);
|
||||
AddParameter(insertCommand, "@callgraph_id", (object?)item.CallgraphId ?? DBNull.Value);
|
||||
AddParameter(insertCommand, "@symbol_id", (object?)item.SymbolId ?? DBNull.Value);
|
||||
AddParameter(insertCommand, "@code_id", (object?)item.CodeId ?? DBNull.Value);
|
||||
AddParameter(insertCommand, "@purl", (object?)item.Purl ?? DBNull.Value);
|
||||
AddParameter(insertCommand, "@edge_from", (object?)item.EdgeFrom ?? DBNull.Value);
|
||||
AddParameter(insertCommand, "@edge_to", (object?)item.EdgeTo ?? DBNull.Value);
|
||||
AddParameter(insertCommand, "@reason", (object?)item.Reason ?? DBNull.Value);
|
||||
AddParameter(insertCommand, "@created_at", item.CreatedAt);
|
||||
|
||||
await insertCommand.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<UnknownSymbolDocument>> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT id, subject_key, callgraph_id, symbol_id, code_id, purl, edge_from, edge_to, reason, created_at
|
||||
FROM signals.unknowns
|
||||
WHERE subject_key = @subject_key
|
||||
ORDER BY created_at DESC";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@subject_key", subjectKey.Trim());
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var results = new List<UnknownSymbolDocument>();
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
results.Add(MapUnknownSymbol(reader));
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
public async Task<int> CountBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(subjectKey);
|
||||
|
||||
await EnsureTableAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
const string sql = @"
|
||||
SELECT COUNT(*)
|
||||
FROM signals.unknowns
|
||||
WHERE subject_key = @subject_key";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(sql, connection);
|
||||
AddParameter(command, "@subject_key", subjectKey.Trim());
|
||||
|
||||
var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
|
||||
return result is long count ? (int)count : 0;
|
||||
}
|
||||
|
||||
private static UnknownSymbolDocument MapUnknownSymbol(NpgsqlDataReader reader) => new()
|
||||
{
|
||||
Id = reader.GetString(0),
|
||||
SubjectKey = reader.GetString(1),
|
||||
CallgraphId = reader.IsDBNull(2) ? null : reader.GetString(2),
|
||||
SymbolId = reader.IsDBNull(3) ? null : reader.GetString(3),
|
||||
CodeId = reader.IsDBNull(4) ? null : reader.GetString(4),
|
||||
Purl = reader.IsDBNull(5) ? null : reader.GetString(5),
|
||||
EdgeFrom = reader.IsDBNull(6) ? null : reader.GetString(6),
|
||||
EdgeTo = reader.IsDBNull(7) ? null : reader.GetString(7),
|
||||
Reason = reader.IsDBNull(8) ? null : reader.GetString(8),
|
||||
CreatedAt = reader.GetFieldValue<DateTimeOffset>(9)
|
||||
};
|
||||
|
||||
private static NpgsqlCommand CreateCommand(string sql, NpgsqlConnection connection, NpgsqlTransaction transaction)
|
||||
{
|
||||
var command = new NpgsqlCommand(sql, connection, transaction);
|
||||
return command;
|
||||
}
|
||||
|
||||
private async Task EnsureTableAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_tableInitialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const string ddl = @"
|
||||
CREATE SCHEMA IF NOT EXISTS signals;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signals.unknowns (
|
||||
id TEXT NOT NULL,
|
||||
subject_key TEXT NOT NULL,
|
||||
callgraph_id TEXT,
|
||||
symbol_id TEXT,
|
||||
code_id TEXT,
|
||||
purl TEXT,
|
||||
edge_from TEXT,
|
||||
edge_to TEXT,
|
||||
reason TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY (subject_key, id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_unknowns_subject_key ON signals.unknowns (subject_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_unknowns_callgraph_id ON signals.unknowns (callgraph_id) WHERE callgraph_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_unknowns_symbol_id ON signals.unknowns (symbol_id) WHERE symbol_id IS NOT NULL;";
|
||||
|
||||
await using var connection = await DataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = CreateCommand(ddl, connection);
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_tableInitialized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Storage.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Postgres;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for configuring Signals PostgreSQL storage services.
|
||||
/// </summary>
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Adds Signals PostgreSQL storage services.
|
||||
/// </summary>
|
||||
/// <param name="services">Service collection.</param>
|
||||
/// <param name="configuration">Configuration root.</param>
|
||||
/// <param name="sectionName">Configuration section name for PostgreSQL options.</param>
|
||||
/// <returns>Service collection for chaining.</returns>
|
||||
public static IServiceCollection AddSignalsPostgresStorage(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration,
|
||||
string sectionName = "Postgres:Signals")
|
||||
{
|
||||
services.Configure<PostgresOptions>(configuration.GetSection(sectionName));
|
||||
services.AddSingleton<SignalsDataSource>();
|
||||
|
||||
// Register repositories
|
||||
services.AddSingleton<ICallgraphRepository, PostgresCallgraphRepository>();
|
||||
services.AddSingleton<IReachabilityFactRepository, PostgresReachabilityFactRepository>();
|
||||
services.AddSingleton<IUnknownsRepository, PostgresUnknownsRepository>();
|
||||
services.AddSingleton<IReachabilityStoreRepository, PostgresReachabilityStoreRepository>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds Signals PostgreSQL storage services with explicit options.
|
||||
/// </summary>
|
||||
/// <param name="services">Service collection.</param>
|
||||
/// <param name="configureOptions">Options configuration action.</param>
|
||||
/// <returns>Service collection for chaining.</returns>
|
||||
public static IServiceCollection AddSignalsPostgresStorage(
|
||||
this IServiceCollection services,
|
||||
Action<PostgresOptions> configureOptions)
|
||||
{
|
||||
services.Configure(configureOptions);
|
||||
services.AddSingleton<SignalsDataSource>();
|
||||
|
||||
// Register repositories
|
||||
services.AddSingleton<ICallgraphRepository, PostgresCallgraphRepository>();
|
||||
services.AddSingleton<IReachabilityFactRepository, PostgresReachabilityFactRepository>();
|
||||
services.AddSingleton<IUnknownsRepository, PostgresUnknownsRepository>();
|
||||
services.AddSingleton<IReachabilityStoreRepository, PostgresReachabilityStoreRepository>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Connections;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Postgres;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL data source for Signals module.
|
||||
/// </summary>
|
||||
public sealed class SignalsDataSource : DataSourceBase
|
||||
{
|
||||
/// <summary>
|
||||
/// Default schema name for Signals tables.
|
||||
/// </summary>
|
||||
public const string DefaultSchemaName = "signals";
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new Signals data source.
|
||||
/// </summary>
|
||||
public SignalsDataSource(IOptions<PostgresOptions> options, ILogger<SignalsDataSource> logger)
|
||||
: base(CreateOptions(options.Value), logger)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override string ModuleName => "Signals";
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void ConfigureDataSourceBuilder(NpgsqlDataSourceBuilder builder)
|
||||
{
|
||||
base.ConfigureDataSourceBuilder(builder);
|
||||
}
|
||||
|
||||
private static PostgresOptions CreateOptions(PostgresOptions baseOptions)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(baseOptions.SchemaName))
|
||||
{
|
||||
baseOptions.SchemaName = DefaultSchemaName;
|
||||
}
|
||||
return baseOptions;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<RootNamespace>StellaOps.Signals.Storage.Postgres</RootNamespace>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Signals/StellaOps.Signals.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Infrastructure.Postgres/StellaOps.Infrastructure.Postgres.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
179
src/Signals/StellaOps.Signals/Models/EdgeBundleDocument.cs
Normal file
179
src/Signals/StellaOps.Signals/Models/EdgeBundleDocument.cs
Normal file
@@ -0,0 +1,179 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Edge bundle document for storing ingested edge bundles.
|
||||
/// </summary>
|
||||
public sealed class EdgeBundleDocument
|
||||
{
|
||||
public string Id { get; set; } = Guid.NewGuid().ToString("N");
|
||||
|
||||
/// <summary>
|
||||
/// Bundle identifier from the DSSE envelope.
|
||||
/// </summary>
|
||||
public string BundleId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Graph hash this bundle is associated with.
|
||||
/// </summary>
|
||||
public string GraphHash { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Tenant identifier for isolation.
|
||||
/// </summary>
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Reason for this bundle (RuntimeHits, InitArray, ThirdParty, Contested, Revoked, etc.).
|
||||
/// </summary>
|
||||
public string BundleReason { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Custom reason description if BundleReason is Custom.
|
||||
/// </summary>
|
||||
public string? CustomReason { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Edges in this bundle.
|
||||
/// </summary>
|
||||
public List<EdgeBundleEdgeDocument> Edges { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Content hash of the bundle (sha256:...).
|
||||
/// </summary>
|
||||
public string ContentHash { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// DSSE envelope digest.
|
||||
/// </summary>
|
||||
public string? DsseDigest { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// CAS URI for the bundle JSON.
|
||||
/// </summary>
|
||||
public string? CasUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// CAS URI for the DSSE envelope.
|
||||
/// </summary>
|
||||
public string? DsseCasUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this bundle has been verified.
|
||||
/// </summary>
|
||||
public bool Verified { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Rekor log index if published.
|
||||
/// </summary>
|
||||
public long? RekorLogIndex { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Count of revoked edges in this bundle.
|
||||
/// </summary>
|
||||
public int RevokedCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the bundle was ingested.
|
||||
/// </summary>
|
||||
public DateTimeOffset IngestedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the bundle was generated.
|
||||
/// </summary>
|
||||
public DateTimeOffset GeneratedAt { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Individual edge within an edge bundle document.
|
||||
/// </summary>
|
||||
public sealed class EdgeBundleEdgeDocument
|
||||
{
|
||||
/// <summary>
|
||||
/// Source function/method ID.
|
||||
/// </summary>
|
||||
public string From { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Target function/method ID.
|
||||
/// </summary>
|
||||
public string To { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Edge kind (call, callvirt, invokestatic, etc.).
|
||||
/// </summary>
|
||||
public string Kind { get; set; } = "call";
|
||||
|
||||
/// <summary>
|
||||
/// Reason for inclusion in this bundle.
|
||||
/// </summary>
|
||||
public string Reason { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether this edge is revoked (patched/removed).
|
||||
/// </summary>
|
||||
public bool Revoked { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Confidence level (0.0-1.0).
|
||||
/// </summary>
|
||||
public double Confidence { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Package URL of the target.
|
||||
/// </summary>
|
||||
public string? Purl { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Symbol digest of the target.
|
||||
/// </summary>
|
||||
public string? SymbolDigest { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Evidence URI for this edge.
|
||||
/// </summary>
|
||||
public string? Evidence { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reference to an edge bundle attached to a reachability fact.
|
||||
/// </summary>
|
||||
public sealed class EdgeBundleReference
|
||||
{
|
||||
/// <summary>
|
||||
/// Bundle identifier.
|
||||
/// </summary>
|
||||
public string BundleId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle reason.
|
||||
/// </summary>
|
||||
public string BundleReason { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// CAS URI for the bundle.
|
||||
/// </summary>
|
||||
public string? CasUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// DSSE CAS URI.
|
||||
/// </summary>
|
||||
public string? DsseCasUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of edges in the bundle.
|
||||
/// </summary>
|
||||
public int EdgeCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of revoked edges.
|
||||
/// </summary>
|
||||
public int RevokedCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the bundle has been verified.
|
||||
/// </summary>
|
||||
public bool Verified { get; set; }
|
||||
}
|
||||
232
src/Signals/StellaOps.Signals/Models/ProcSnapshotDocument.cs
Normal file
232
src/Signals/StellaOps.Signals/Models/ProcSnapshotDocument.cs
Normal file
@@ -0,0 +1,232 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Document representing a proc snapshot for Java/.NET/PHP runtime parity.
|
||||
/// Captures runtime-observed classpath, loaded assemblies, and autoload paths.
|
||||
/// </summary>
|
||||
public sealed class ProcSnapshotDocument
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique identifier for this snapshot (format: {tenant}/{image_digest}/{snapshot_hash}).
|
||||
/// </summary>
|
||||
public required string Id { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Tenant identifier for multi-tenancy isolation.
|
||||
/// </summary>
|
||||
public required string Tenant { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Image digest of the container this snapshot was captured from.
|
||||
/// </summary>
|
||||
[JsonPropertyName("imageDigest")]
|
||||
public required string ImageDigest { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Node identifier where the snapshot was captured.
|
||||
/// </summary>
|
||||
public string? Node { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Container ID where the process was running.
|
||||
/// </summary>
|
||||
public string? ContainerId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Process ID at capture time.
|
||||
/// </summary>
|
||||
public int Pid { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Process entrypoint command.
|
||||
/// </summary>
|
||||
public string? Entrypoint { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Runtime type: java, dotnet, php.
|
||||
/// </summary>
|
||||
public required string RuntimeType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Runtime version (e.g., "17.0.2", "8.0.0", "8.2.0").
|
||||
/// </summary>
|
||||
public string? RuntimeVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Java classpath entries (jar paths, directories).
|
||||
/// </summary>
|
||||
[JsonPropertyName("classpath")]
|
||||
public IReadOnlyList<ClasspathEntry> Classpath { get; init; } = Array.Empty<ClasspathEntry>();
|
||||
|
||||
/// <summary>
|
||||
/// .NET loaded assemblies with RID-graph resolution.
|
||||
/// </summary>
|
||||
[JsonPropertyName("loadedAssemblies")]
|
||||
public IReadOnlyList<LoadedAssemblyEntry> LoadedAssemblies { get; init; } = Array.Empty<LoadedAssemblyEntry>();
|
||||
|
||||
/// <summary>
|
||||
/// PHP autoload paths from composer autoloader.
|
||||
/// </summary>
|
||||
[JsonPropertyName("autoloadPaths")]
|
||||
public IReadOnlyList<AutoloadPathEntry> AutoloadPaths { get; init; } = Array.Empty<AutoloadPathEntry>();
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp when the snapshot was captured.
|
||||
/// </summary>
|
||||
public DateTimeOffset CapturedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp when the snapshot was stored.
|
||||
/// </summary>
|
||||
public DateTimeOffset StoredAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Expiration timestamp for TTL-based cleanup.
|
||||
/// </summary>
|
||||
public DateTimeOffset? ExpiresAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional annotations/metadata.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, string>? Annotations { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Java classpath entry representing a JAR or directory.
|
||||
/// </summary>
|
||||
public sealed class ClasspathEntry
|
||||
{
|
||||
/// <summary>
|
||||
/// Path to the JAR file or classpath directory.
|
||||
/// </summary>
|
||||
public required string Path { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Type: jar, directory, jmod.
|
||||
/// </summary>
|
||||
public string? Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the JAR file (null for directories).
|
||||
/// </summary>
|
||||
public string? Sha256 { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Maven coordinate if resolvable (e.g., "org.springframework:spring-core:5.3.20").
|
||||
/// </summary>
|
||||
public string? MavenCoordinate { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Package URL (PURL) if resolvable.
|
||||
/// </summary>
|
||||
public string? Purl { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Size in bytes.
|
||||
/// </summary>
|
||||
public long? SizeBytes { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// .NET loaded assembly entry with RID-graph context.
|
||||
/// </summary>
|
||||
public sealed class LoadedAssemblyEntry
|
||||
{
|
||||
/// <summary>
|
||||
/// Assembly name (e.g., "System.Text.Json").
|
||||
/// </summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Assembly version (e.g., "8.0.0.0").
|
||||
/// </summary>
|
||||
public string? Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Full path to the loaded DLL.
|
||||
/// </summary>
|
||||
public required string Path { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the assembly file.
|
||||
/// </summary>
|
||||
public string? Sha256 { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// NuGet package ID if resolvable.
|
||||
/// </summary>
|
||||
public string? NuGetPackage { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// NuGet package version if resolvable.
|
||||
/// </summary>
|
||||
public string? NuGetVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Package URL (PURL) if resolvable.
|
||||
/// </summary>
|
||||
public string? Purl { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Runtime identifier (RID) for platform-specific assemblies.
|
||||
/// </summary>
|
||||
public string? Rid { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this assembly was loaded from the shared framework.
|
||||
/// </summary>
|
||||
public bool? IsFrameworkAssembly { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Source from deps.json resolution: compile, runtime, native.
|
||||
/// </summary>
|
||||
public string? DepsSource { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PHP autoload path entry from Composer.
|
||||
/// </summary>
|
||||
public sealed class AutoloadPathEntry
|
||||
{
|
||||
/// <summary>
|
||||
/// Namespace prefix (PSR-4) or class name (classmap).
|
||||
/// </summary>
|
||||
public string? Namespace { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Autoload type: psr-4, psr-0, classmap, files.
|
||||
/// </summary>
|
||||
public required string Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to the autoloaded file or directory.
|
||||
/// </summary>
|
||||
public required string Path { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Composer package name if resolvable.
|
||||
/// </summary>
|
||||
public string? ComposerPackage { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Composer package version if resolvable.
|
||||
/// </summary>
|
||||
public string? ComposerVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Package URL (PURL) if resolvable.
|
||||
/// </summary>
|
||||
public string? Purl { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Known runtime types for proc snapshots.
|
||||
/// </summary>
|
||||
public static class ProcSnapshotRuntimeTypes
|
||||
{
|
||||
public const string Java = "java";
|
||||
public const string DotNet = "dotnet";
|
||||
public const string Php = "php";
|
||||
}
|
||||
@@ -17,12 +17,32 @@ public sealed class ReachabilityFactDocument
|
||||
|
||||
public List<RuntimeFactDocument>? RuntimeFacts { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// CAS URI for the runtime-facts batch artifact (cas://reachability/runtime-facts/{hash}).
|
||||
/// </summary>
|
||||
public string? RuntimeFactsBatchUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// BLAKE3 hash of the runtime-facts batch artifact.
|
||||
/// </summary>
|
||||
public string? RuntimeFactsBatchHash { get; set; }
|
||||
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
|
||||
public ContextFacts? ContextFacts { get; set; }
|
||||
|
||||
public UncertaintyDocument? Uncertainty { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Edge bundles attached to this graph.
|
||||
/// </summary>
|
||||
public List<EdgeBundleReference>? EdgeBundles { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether any edges are quarantined (revoked) for this fact.
|
||||
/// </summary>
|
||||
public bool HasQuarantinedEdges { get; set; }
|
||||
|
||||
public double Score { get; set; }
|
||||
|
||||
public double RiskScore { get; set; }
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Persistence;
|
||||
|
||||
/// <summary>
|
||||
/// Repository for persisting and querying proc snapshot documents.
|
||||
/// </summary>
|
||||
public interface IProcSnapshotRepository
|
||||
{
|
||||
/// <summary>
|
||||
/// Upsert a proc snapshot document.
|
||||
/// </summary>
|
||||
Task<ProcSnapshotDocument> UpsertAsync(ProcSnapshotDocument document, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Get a proc snapshot by ID.
|
||||
/// </summary>
|
||||
Task<ProcSnapshotDocument?> GetByIdAsync(string id, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Get all proc snapshots for a specific image digest.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<ProcSnapshotDocument>> GetByImageDigestAsync(
|
||||
string tenant,
|
||||
string imageDigest,
|
||||
int limit = 100,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Get the most recent proc snapshot for an image digest and runtime type.
|
||||
/// </summary>
|
||||
Task<ProcSnapshotDocument?> GetLatestAsync(
|
||||
string tenant,
|
||||
string imageDigest,
|
||||
string runtimeType,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Delete expired proc snapshots.
|
||||
/// </summary>
|
||||
Task<int> DeleteExpiredAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
using System.Collections.Concurrent;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Persistence;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of <see cref="IProcSnapshotRepository"/> for testing and development.
|
||||
/// </summary>
|
||||
public sealed class InMemoryProcSnapshotRepository : IProcSnapshotRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ProcSnapshotDocument> _documents = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public InMemoryProcSnapshotRepository(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public Task<ProcSnapshotDocument> UpsertAsync(ProcSnapshotDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
_documents[document.Id] = document;
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
|
||||
public Task<ProcSnapshotDocument?> GetByIdAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(id);
|
||||
|
||||
_documents.TryGetValue(id, out var document);
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<ProcSnapshotDocument>> GetByImageDigestAsync(
|
||||
string tenant,
|
||||
string imageDigest,
|
||||
int limit = 100,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
|
||||
|
||||
var normalizedDigest = imageDigest.ToLowerInvariant();
|
||||
var results = _documents.Values
|
||||
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(d.ImageDigest, normalizedDigest, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderByDescending(d => d.CapturedAt)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<ProcSnapshotDocument>>(results);
|
||||
}
|
||||
|
||||
public Task<ProcSnapshotDocument?> GetLatestAsync(
|
||||
string tenant,
|
||||
string imageDigest,
|
||||
string runtimeType,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runtimeType);
|
||||
|
||||
var normalizedDigest = imageDigest.ToLowerInvariant();
|
||||
var result = _documents.Values
|
||||
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(d.ImageDigest, normalizedDigest, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(d.RuntimeType, runtimeType, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderByDescending(d => d.CapturedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public Task<int> DeleteExpiredAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var expiredIds = _documents
|
||||
.Where(kv => kv.Value.ExpiresAt.HasValue && kv.Value.ExpiresAt.Value < now)
|
||||
.Select(kv => kv.Key)
|
||||
.ToList();
|
||||
|
||||
var deletedCount = 0;
|
||||
foreach (var id in expiredIds)
|
||||
{
|
||||
if (_documents.TryRemove(id, out _))
|
||||
{
|
||||
deletedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult(deletedCount);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Ingests edge-bundle DSSE envelopes, attaches to graph_hash, enforces quarantine for revoked edges.
|
||||
/// </summary>
|
||||
public sealed class EdgeBundleIngestionService : IEdgeBundleIngestionService
|
||||
{
|
||||
private readonly ILogger<EdgeBundleIngestionService> _logger;
|
||||
private readonly SignalsOptions _options;
|
||||
|
||||
// In-memory storage (in production, would use repository)
|
||||
private readonly ConcurrentDictionary<string, List<EdgeBundleDocument>> _bundlesByGraphHash = new();
|
||||
private readonly ConcurrentDictionary<string, HashSet<string>> _revokedEdgeKeys = new();
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
public EdgeBundleIngestionService(
|
||||
ILogger<EdgeBundleIngestionService> logger,
|
||||
IOptions<SignalsOptions> options)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
public async Task<EdgeBundleIngestResponse> IngestAsync(
|
||||
string tenantId,
|
||||
Stream bundleStream,
|
||||
Stream? dsseStream,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(bundleStream);
|
||||
|
||||
// Parse the bundle JSON
|
||||
using var bundleMs = new MemoryStream();
|
||||
await bundleStream.CopyToAsync(bundleMs, cancellationToken).ConfigureAwait(false);
|
||||
bundleMs.Position = 0;
|
||||
|
||||
var bundleJson = await JsonDocument.ParseAsync(bundleMs, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var root = bundleJson.RootElement;
|
||||
|
||||
// Extract bundle fields
|
||||
var bundleId = GetStringOrDefault(root, "bundleId", $"bundle:{Guid.NewGuid():N}");
|
||||
var graphHash = GetStringOrDefault(root, "graphHash", string.Empty);
|
||||
var bundleReason = GetStringOrDefault(root, "bundleReason", "Custom");
|
||||
var customReason = GetStringOrDefault(root, "customReason", null);
|
||||
var generatedAtStr = GetStringOrDefault(root, "generatedAt", null);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(graphHash))
|
||||
{
|
||||
throw new InvalidOperationException("Edge bundle missing required 'graphHash' field");
|
||||
}
|
||||
|
||||
var generatedAt = !string.IsNullOrWhiteSpace(generatedAtStr)
|
||||
? DateTimeOffset.Parse(generatedAtStr)
|
||||
: DateTimeOffset.UtcNow;
|
||||
|
||||
// Parse edges
|
||||
var edges = new List<EdgeBundleEdgeDocument>();
|
||||
var revokedCount = 0;
|
||||
|
||||
if (root.TryGetProperty("edges", out var edgesElement) && edgesElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var edgeEl in edgesElement.EnumerateArray())
|
||||
{
|
||||
var edge = new EdgeBundleEdgeDocument
|
||||
{
|
||||
From = GetStringOrDefault(edgeEl, "from", string.Empty),
|
||||
To = GetStringOrDefault(edgeEl, "to", string.Empty),
|
||||
Kind = GetStringOrDefault(edgeEl, "kind", "call"),
|
||||
Reason = GetStringOrDefault(edgeEl, "reason", "Unknown"),
|
||||
Revoked = edgeEl.TryGetProperty("revoked", out var r) && r.GetBoolean(),
|
||||
Confidence = edgeEl.TryGetProperty("confidence", out var c) ? c.GetDouble() : 0.5,
|
||||
Purl = GetStringOrDefault(edgeEl, "purl", null),
|
||||
SymbolDigest = GetStringOrDefault(edgeEl, "symbolDigest", null),
|
||||
Evidence = GetStringOrDefault(edgeEl, "evidence", null)
|
||||
};
|
||||
|
||||
edges.Add(edge);
|
||||
|
||||
if (edge.Revoked)
|
||||
{
|
||||
revokedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute content hash
|
||||
bundleMs.Position = 0;
|
||||
var contentHash = ComputeSha256(bundleMs);
|
||||
|
||||
// Parse DSSE if provided
|
||||
string? dsseDigest = null;
|
||||
if (dsseStream is not null)
|
||||
{
|
||||
using var dsseMs = new MemoryStream();
|
||||
await dsseStream.CopyToAsync(dsseMs, cancellationToken).ConfigureAwait(false);
|
||||
dsseMs.Position = 0;
|
||||
dsseDigest = $"sha256:{ComputeSha256(dsseMs)}";
|
||||
}
|
||||
|
||||
// Build CAS URIs
|
||||
var graphHashDigest = ExtractHashDigest(graphHash);
|
||||
var casUri = $"cas://reachability/edges/{graphHashDigest}/{bundleId}";
|
||||
var dsseCasUri = dsseStream is not null ? $"{casUri}.dsse" : null;
|
||||
|
||||
// Create document
|
||||
var document = new EdgeBundleDocument
|
||||
{
|
||||
BundleId = bundleId,
|
||||
GraphHash = graphHash,
|
||||
TenantId = tenantId,
|
||||
BundleReason = bundleReason,
|
||||
CustomReason = customReason,
|
||||
Edges = edges,
|
||||
ContentHash = $"sha256:{contentHash}",
|
||||
DsseDigest = dsseDigest,
|
||||
CasUri = casUri,
|
||||
DsseCasUri = dsseCasUri,
|
||||
Verified = dsseStream is not null, // Simple verification - in production would verify signature
|
||||
RevokedCount = revokedCount,
|
||||
IngestedAt = DateTimeOffset.UtcNow,
|
||||
GeneratedAt = generatedAt
|
||||
};
|
||||
|
||||
// Store document
|
||||
var storageKey = $"{tenantId}:{graphHash}";
|
||||
_bundlesByGraphHash.AddOrUpdate(
|
||||
storageKey,
|
||||
_ => new List<EdgeBundleDocument> { document },
|
||||
(_, list) =>
|
||||
{
|
||||
// Remove existing bundle with same ID
|
||||
list.RemoveAll(b => b.BundleId == bundleId);
|
||||
list.Add(document);
|
||||
return list;
|
||||
});
|
||||
|
||||
// Update revoked edge index for quarantine enforcement
|
||||
if (revokedCount > 0)
|
||||
{
|
||||
var revokedEdges = edges.Where(e => e.Revoked).Select(e => $"{e.From}>{e.To}").ToHashSet();
|
||||
_revokedEdgeKeys.AddOrUpdate(
|
||||
storageKey,
|
||||
_ => revokedEdges,
|
||||
(_, existing) =>
|
||||
{
|
||||
foreach (var key in revokedEdges)
|
||||
{
|
||||
existing.Add(key);
|
||||
}
|
||||
return existing;
|
||||
});
|
||||
}
|
||||
|
||||
var quarantined = revokedCount > 0;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Ingested edge bundle {BundleId} for graph {GraphHash} with {EdgeCount} edges ({RevokedCount} revoked, quarantine={Quarantined})",
|
||||
bundleId, graphHash, edges.Count, revokedCount, quarantined);
|
||||
|
||||
return new EdgeBundleIngestResponse(
|
||||
bundleId,
|
||||
graphHash,
|
||||
bundleReason,
|
||||
casUri,
|
||||
dsseCasUri,
|
||||
edges.Count,
|
||||
revokedCount,
|
||||
quarantined);
|
||||
}
|
||||
|
||||
public Task<EdgeBundleDocument[]> GetBundlesForGraphAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = $"{tenantId}:{graphHash}";
|
||||
if (_bundlesByGraphHash.TryGetValue(key, out var bundles))
|
||||
{
|
||||
return Task.FromResult(bundles.ToArray());
|
||||
}
|
||||
|
||||
return Task.FromResult(Array.Empty<EdgeBundleDocument>());
|
||||
}
|
||||
|
||||
public Task<EdgeBundleEdgeDocument[]> GetRevokedEdgesAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = $"{tenantId}:{graphHash}";
|
||||
if (_bundlesByGraphHash.TryGetValue(key, out var bundles))
|
||||
{
|
||||
var revoked = bundles
|
||||
.SelectMany(b => b.Edges)
|
||||
.Where(e => e.Revoked)
|
||||
.ToArray();
|
||||
return Task.FromResult(revoked);
|
||||
}
|
||||
|
||||
return Task.FromResult(Array.Empty<EdgeBundleEdgeDocument>());
|
||||
}
|
||||
|
||||
public Task<bool> IsEdgeRevokedAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
string fromId,
|
||||
string toId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = $"{tenantId}:{graphHash}";
|
||||
if (_revokedEdgeKeys.TryGetValue(key, out var revokedKeys))
|
||||
{
|
||||
var edgeKey = $"{fromId}>{toId}";
|
||||
return Task.FromResult(revokedKeys.Contains(edgeKey));
|
||||
}
|
||||
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
private static string GetStringOrDefault(JsonElement element, string propertyName, string? defaultValue)
|
||||
{
|
||||
if (element.TryGetProperty(propertyName, out var prop) && prop.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return prop.GetString() ?? defaultValue ?? string.Empty;
|
||||
}
|
||||
return defaultValue ?? string.Empty;
|
||||
}
|
||||
|
||||
private static string ComputeSha256(Stream stream)
|
||||
{
|
||||
using var sha = SHA256.Create();
|
||||
var hash = sha.ComputeHash(stream);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string ExtractHashDigest(string prefixedHash)
|
||||
{
|
||||
var colonIndex = prefixedHash.IndexOf(':');
|
||||
return colonIndex >= 0 ? prefixedHash[(colonIndex + 1)..] : prefixedHash;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Response from edge bundle ingestion.
|
||||
/// </summary>
|
||||
public sealed record EdgeBundleIngestResponse(
|
||||
string BundleId,
|
||||
string GraphHash,
|
||||
string BundleReason,
|
||||
string CasUri,
|
||||
string? DsseCasUri,
|
||||
int EdgeCount,
|
||||
int RevokedCount,
|
||||
bool Quarantined);
|
||||
|
||||
/// <summary>
|
||||
/// Service for ingesting edge-bundle DSSE envelopes.
|
||||
/// </summary>
|
||||
public interface IEdgeBundleIngestionService
|
||||
{
|
||||
/// <summary>
|
||||
/// Ingests an edge bundle from a JSON stream.
|
||||
/// </summary>
|
||||
/// <param name="tenantId">Tenant identifier for isolation.</param>
|
||||
/// <param name="bundleStream">Stream containing the edge-bundle JSON.</param>
|
||||
/// <param name="dsseStream">Optional stream containing the DSSE envelope.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Ingest response with bundle details.</returns>
|
||||
Task<EdgeBundleIngestResponse> IngestAsync(
|
||||
string tenantId,
|
||||
Stream bundleStream,
|
||||
Stream? dsseStream,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets all edge bundles for a graph hash.
|
||||
/// </summary>
|
||||
Task<EdgeBundleDocument[]> GetBundlesForGraphAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets revoked edges from all bundles for a graph.
|
||||
/// Returns edges that should be quarantined from scoring.
|
||||
/// </summary>
|
||||
Task<EdgeBundleEdgeDocument[]> GetRevokedEdgesAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if an edge is revoked for the given graph.
|
||||
/// </summary>
|
||||
Task<bool> IsEdgeRevokedAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
string fromId,
|
||||
string toId,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -1,10 +1,66 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
public interface IRuntimeFactsIngestionService
|
||||
{
|
||||
/// <summary>
|
||||
/// Ingests runtime facts from a structured request.
|
||||
/// </summary>
|
||||
Task<RuntimeFactsIngestResponse> IngestAsync(RuntimeFactsIngestRequest request, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Ingests runtime facts from a raw NDJSON/gzip stream, stores in CAS, and processes.
|
||||
/// </summary>
|
||||
/// <param name="tenantId">Tenant identifier for tenant isolation.</param>
|
||||
/// <param name="content">The NDJSON or gzip compressed stream of runtime fact events.</param>
|
||||
/// <param name="contentType">Content type (application/x-ndjson or application/gzip).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Batch ingestion response with CAS reference.</returns>
|
||||
Task<RuntimeFactsBatchIngestResponse> IngestBatchAsync(
|
||||
string tenantId,
|
||||
Stream content,
|
||||
string contentType,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Response from batch ingestion with CAS storage.
|
||||
/// </summary>
|
||||
public sealed record RuntimeFactsBatchIngestResponse
|
||||
{
|
||||
/// <summary>
|
||||
/// CAS URI for the stored batch artifact.
|
||||
/// </summary>
|
||||
public required string CasUri { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// BLAKE3 hash of the batch artifact.
|
||||
/// </summary>
|
||||
public required string BatchHash { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of fact documents processed.
|
||||
/// </summary>
|
||||
public int ProcessedCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total events ingested.
|
||||
/// </summary>
|
||||
public int TotalEvents { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total hit count across all events.
|
||||
/// </summary>
|
||||
public long TotalHitCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Subject keys affected.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> SubjectKeys { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp of ingestion.
|
||||
/// </summary>
|
||||
public DateTimeOffset StoredAt { get; init; }
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Cryptography;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Storage;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
@@ -18,6 +18,8 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
private readonly IEventsPublisher eventsPublisher;
|
||||
private readonly IReachabilityScoringService scoringService;
|
||||
private readonly IRuntimeFactsProvenanceNormalizer provenanceNormalizer;
|
||||
private readonly IRuntimeFactsArtifactStore? artifactStore;
|
||||
private readonly ICryptoHash? cryptoHash;
|
||||
private readonly ILogger<RuntimeFactsIngestionService> logger;
|
||||
|
||||
public RuntimeFactsIngestionService(
|
||||
@@ -27,7 +29,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
IEventsPublisher eventsPublisher,
|
||||
IReachabilityScoringService scoringService,
|
||||
IRuntimeFactsProvenanceNormalizer provenanceNormalizer,
|
||||
ILogger<RuntimeFactsIngestionService> logger)
|
||||
ILogger<RuntimeFactsIngestionService> logger,
|
||||
IRuntimeFactsArtifactStore? artifactStore = null,
|
||||
ICryptoHash? cryptoHash = null)
|
||||
{
|
||||
this.factRepository = factRepository ?? throw new ArgumentNullException(nameof(factRepository));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
@@ -35,6 +39,8 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
this.eventsPublisher = eventsPublisher ?? throw new ArgumentNullException(nameof(eventsPublisher));
|
||||
this.scoringService = scoringService ?? throw new ArgumentNullException(nameof(scoringService));
|
||||
this.provenanceNormalizer = provenanceNormalizer ?? throw new ArgumentNullException(nameof(provenanceNormalizer));
|
||||
this.artifactStore = artifactStore;
|
||||
this.cryptoHash = cryptoHash;
|
||||
this.logger = logger ?? NullLogger<RuntimeFactsIngestionService>.Instance;
|
||||
}
|
||||
|
||||
@@ -96,6 +102,216 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<RuntimeFactsBatchIngestResponse> IngestBatchAsync(
|
||||
string tenantId,
|
||||
Stream content,
|
||||
string contentType,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var storedAt = timeProvider.GetUtcNow();
|
||||
var subjectKeys = new HashSet<string>(StringComparer.Ordinal);
|
||||
var processedCount = 0;
|
||||
var totalEvents = 0;
|
||||
long totalHitCount = 0;
|
||||
|
||||
// Buffer the content for hashing and parsing
|
||||
using var buffer = new MemoryStream();
|
||||
await content.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
|
||||
// Compute BLAKE3 hash
|
||||
string batchHash;
|
||||
if (cryptoHash != null)
|
||||
{
|
||||
batchHash = "blake3:" + await cryptoHash.ComputeHashHexAsync(buffer, "BLAKE3-256", cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fallback: generate a deterministic hash based on content length and timestamp
|
||||
batchHash = $"blake3:{storedAt.ToUnixTimeMilliseconds():x16}{buffer.Length:x16}";
|
||||
}
|
||||
|
||||
// Store to CAS if artifact store is available
|
||||
StoredRuntimeFactsArtifact? storedArtifact = null;
|
||||
if (artifactStore != null)
|
||||
{
|
||||
var fileName = contentType.Contains("gzip", StringComparison.OrdinalIgnoreCase)
|
||||
? "runtime-facts.ndjson.gz"
|
||||
: "runtime-facts.ndjson";
|
||||
|
||||
var saveRequest = new RuntimeFactsArtifactSaveRequest(
|
||||
TenantId: tenantId,
|
||||
SubjectKey: string.Empty, // Will be populated after parsing
|
||||
Hash: batchHash.Replace("blake3:", string.Empty),
|
||||
ContentType: contentType,
|
||||
FileName: fileName,
|
||||
BatchSize: buffer.Length,
|
||||
ProvenanceSource: "runtime-facts-batch");
|
||||
|
||||
storedArtifact = await artifactStore.SaveAsync(saveRequest, buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
}
|
||||
|
||||
// Decompress if gzip
|
||||
Stream parseStream;
|
||||
if (contentType.Contains("gzip", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var decompressed = new MemoryStream();
|
||||
await using (var gzip = new GZipStream(buffer, CompressionMode.Decompress, leaveOpen: true))
|
||||
{
|
||||
await gzip.CopyToAsync(decompressed, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
decompressed.Position = 0;
|
||||
parseStream = decompressed;
|
||||
}
|
||||
else
|
||||
{
|
||||
parseStream = buffer;
|
||||
}
|
||||
|
||||
// Parse NDJSON and group by subject
|
||||
var requestsBySubject = new Dictionary<string, RuntimeFactsIngestRequest>(StringComparer.Ordinal);
|
||||
using var reader = new StreamReader(parseStream, leaveOpen: true);
|
||||
|
||||
while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var evt = JsonSerializer.Deserialize<RuntimeFactsBatchEvent>(line, JsonOptions);
|
||||
if (evt is null || string.IsNullOrWhiteSpace(evt.SymbolId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var subjectKey = evt.Subject?.ToSubjectKey() ?? evt.CallgraphId ?? "unknown";
|
||||
if (!requestsBySubject.TryGetValue(subjectKey, out var request))
|
||||
{
|
||||
request = new RuntimeFactsIngestRequest
|
||||
{
|
||||
Subject = evt.Subject ?? new ReachabilitySubject { ScanId = subjectKey },
|
||||
CallgraphId = evt.CallgraphId ?? subjectKey,
|
||||
Events = new List<RuntimeFactEvent>(),
|
||||
Metadata = new Dictionary<string, string?>(StringComparer.Ordinal)
|
||||
{
|
||||
["batch.hash"] = batchHash,
|
||||
["batch.cas_uri"] = storedArtifact?.CasUri,
|
||||
["tenant_id"] = tenantId
|
||||
}
|
||||
};
|
||||
requestsBySubject[subjectKey] = request;
|
||||
}
|
||||
|
||||
((List<RuntimeFactEvent>)request.Events).Add(new RuntimeFactEvent
|
||||
{
|
||||
SymbolId = evt.SymbolId,
|
||||
CodeId = evt.CodeId,
|
||||
SymbolDigest = evt.SymbolDigest,
|
||||
Purl = evt.Purl,
|
||||
BuildId = evt.BuildId,
|
||||
LoaderBase = evt.LoaderBase,
|
||||
ProcessId = evt.ProcessId,
|
||||
ProcessName = evt.ProcessName,
|
||||
SocketAddress = evt.SocketAddress,
|
||||
ContainerId = evt.ContainerId,
|
||||
EvidenceUri = evt.EvidenceUri,
|
||||
HitCount = Math.Max(evt.HitCount, 1),
|
||||
ObservedAt = evt.ObservedAt,
|
||||
Metadata = evt.Metadata
|
||||
});
|
||||
|
||||
totalEvents++;
|
||||
totalHitCount += Math.Max(evt.HitCount, 1);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Failed to parse NDJSON line in batch ingestion.");
|
||||
}
|
||||
}
|
||||
|
||||
// Process each subject's request
|
||||
foreach (var (subjectKey, request) in requestsBySubject)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await IngestAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Update the fact document with batch reference
|
||||
var existing = await factRepository.GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
if (existing != null && storedArtifact != null)
|
||||
{
|
||||
existing.RuntimeFactsBatchUri = storedArtifact.CasUri;
|
||||
existing.RuntimeFactsBatchHash = batchHash;
|
||||
await factRepository.UpsertAsync(existing, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
subjectKeys.Add(subjectKey);
|
||||
processedCount++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to ingest batch for subject {SubjectKey}.", subjectKey);
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Batch ingestion completed: {ProcessedCount} subjects, {TotalEvents} events, {TotalHitCount} hits (hash={BatchHash}, tenant={TenantId}).",
|
||||
processedCount,
|
||||
totalEvents,
|
||||
totalHitCount,
|
||||
batchHash,
|
||||
tenantId);
|
||||
|
||||
return new RuntimeFactsBatchIngestResponse
|
||||
{
|
||||
CasUri = storedArtifact?.CasUri ?? $"cas://reachability/runtime-facts/{batchHash.Replace("blake3:", string.Empty)}",
|
||||
BatchHash = batchHash,
|
||||
ProcessedCount = processedCount,
|
||||
TotalEvents = totalEvents,
|
||||
TotalHitCount = totalHitCount,
|
||||
SubjectKeys = subjectKeys.ToList(),
|
||||
StoredAt = storedAt
|
||||
};
|
||||
}
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// NDJSON batch event structure for runtime facts.
|
||||
/// </summary>
|
||||
private sealed class RuntimeFactsBatchEvent
|
||||
{
|
||||
public string? SymbolId { get; set; }
|
||||
public string? CodeId { get; set; }
|
||||
public string? SymbolDigest { get; set; }
|
||||
public string? Purl { get; set; }
|
||||
public string? BuildId { get; set; }
|
||||
public string? LoaderBase { get; set; }
|
||||
public int? ProcessId { get; set; }
|
||||
public string? ProcessName { get; set; }
|
||||
public string? SocketAddress { get; set; }
|
||||
public string? ContainerId { get; set; }
|
||||
public string? EvidenceUri { get; set; }
|
||||
public int HitCount { get; set; } = 1;
|
||||
public DateTimeOffset? ObservedAt { get; set; }
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
public ReachabilitySubject? Subject { get; set; }
|
||||
public string? CallgraphId { get; set; }
|
||||
}
|
||||
|
||||
private static void ValidateRequest(RuntimeFactsIngestRequest request)
|
||||
{
|
||||
if (request.Subject is null)
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Stores runtime-facts batch artifacts on the local filesystem.
|
||||
/// CAS paths: cas://reachability/runtime-facts/{hash}
|
||||
/// </summary>
|
||||
internal sealed class FileSystemRuntimeFactsArtifactStore : IRuntimeFactsArtifactStore
|
||||
{
|
||||
private const string DefaultFileName = "runtime-facts.ndjson";
|
||||
|
||||
private readonly SignalsArtifactStorageOptions _storageOptions;
|
||||
private readonly ILogger<FileSystemRuntimeFactsArtifactStore> _logger;
|
||||
|
||||
public FileSystemRuntimeFactsArtifactStore(
|
||||
IOptions<SignalsOptions> options,
|
||||
ILogger<FileSystemRuntimeFactsArtifactStore> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_storageOptions = options.Value.Storage;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<StoredRuntimeFactsArtifact> SaveAsync(
|
||||
RuntimeFactsArtifactSaveRequest request,
|
||||
Stream content,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var hash = NormalizeHash(request.Hash);
|
||||
if (string.IsNullOrWhiteSpace(hash))
|
||||
{
|
||||
throw new InvalidOperationException("Runtime-facts artifact hash is required for CAS storage.");
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(hash);
|
||||
Directory.CreateDirectory(casDirectory);
|
||||
|
||||
var fileName = SanitizeFileName(string.IsNullOrWhiteSpace(request.FileName) ? DefaultFileName : request.FileName);
|
||||
var destinationPath = Path.Combine(casDirectory, fileName);
|
||||
|
||||
await using (var fileStream = File.Create(destinationPath))
|
||||
{
|
||||
await content.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var fileInfo = new FileInfo(destinationPath);
|
||||
var casUri = $"cas://reachability/runtime-facts/{hash}";
|
||||
|
||||
_logger.LogInformation(
|
||||
"Stored runtime-facts artifact at {Path} (length={Length}, hash={Hash}, tenant={TenantId}).",
|
||||
destinationPath,
|
||||
fileInfo.Length,
|
||||
hash,
|
||||
request.TenantId);
|
||||
|
||||
return new StoredRuntimeFactsArtifact(
|
||||
Path.GetRelativePath(_storageOptions.RootPath, destinationPath),
|
||||
fileInfo.Length,
|
||||
hash,
|
||||
request.ContentType,
|
||||
casUri);
|
||||
}
|
||||
|
||||
public Task<Stream?> GetAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
var filePath = Path.Combine(casDirectory, DefaultFileName);
|
||||
|
||||
// Also check for gzip variant
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
filePath = Path.Combine(casDirectory, "runtime-facts.ndjson.gz");
|
||||
}
|
||||
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
_logger.LogDebug("Runtime-facts artifact {Hash} not found at {Path}.", normalizedHash, filePath);
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
var content = new MemoryStream();
|
||||
using (var fileStream = File.OpenRead(filePath))
|
||||
{
|
||||
fileStream.CopyTo(content);
|
||||
}
|
||||
|
||||
content.Position = 0;
|
||||
_logger.LogDebug("Retrieved runtime-facts artifact {Hash} from {Path}.", normalizedHash, filePath);
|
||||
return Task.FromResult<Stream?>(content);
|
||||
}
|
||||
|
||||
public Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
var defaultPath = Path.Combine(casDirectory, DefaultFileName);
|
||||
var gzipPath = Path.Combine(casDirectory, "runtime-facts.ndjson.gz");
|
||||
var exists = File.Exists(defaultPath) || File.Exists(gzipPath);
|
||||
|
||||
_logger.LogDebug("Runtime-facts artifact {Hash} exists={Exists}.", normalizedHash, exists);
|
||||
return Task.FromResult(exists);
|
||||
}
|
||||
|
||||
public Task<bool> DeleteAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
if (!Directory.Exists(casDirectory))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(casDirectory, recursive: true);
|
||||
_logger.LogInformation("Deleted runtime-facts artifact {Hash}.", normalizedHash);
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete runtime-facts artifact {Hash}.", normalizedHash);
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
}
|
||||
|
||||
private string GetCasDirectory(string hash)
|
||||
{
|
||||
var prefix = hash.Length >= 2 ? hash[..2] : hash;
|
||||
return Path.Combine(_storageOptions.RootPath, "cas", "reachability", "runtime-facts", prefix, hash);
|
||||
}
|
||||
|
||||
private static string? NormalizeHash(string? hash)
|
||||
=> hash?.Trim().ToLowerInvariant();
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
=> string.Join('_', value.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)).ToLowerInvariant();
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Persists and retrieves runtime-facts batch artifacts from content-addressable storage.
|
||||
/// CAS paths follow: cas://reachability/runtime-facts/{hash}
|
||||
/// </summary>
|
||||
public interface IRuntimeFactsArtifactStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Stores a runtime-facts batch artifact.
|
||||
/// </summary>
|
||||
/// <param name="request">Metadata about the artifact to store.</param>
|
||||
/// <param name="content">The artifact content stream (NDJSON or gzip compressed).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Information about the stored artifact including CAS URI.</returns>
|
||||
Task<StoredRuntimeFactsArtifact> SaveAsync(
|
||||
RuntimeFactsArtifactSaveRequest request,
|
||||
Stream content,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a runtime-facts artifact by its BLAKE3 hash.
|
||||
/// </summary>
|
||||
/// <param name="hash">The BLAKE3 hash of the artifact (blake3:{hex}).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The artifact content stream, or null if not found.</returns>
|
||||
Task<Stream?> GetAsync(string hash, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a runtime-facts artifact exists.
|
||||
/// </summary>
|
||||
/// <param name="hash">The BLAKE3 hash of the artifact.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>True if the artifact exists.</returns>
|
||||
Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a runtime-facts artifact if it exists.
|
||||
/// </summary>
|
||||
/// <param name="hash">The BLAKE3 hash of the artifact.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>True if the artifact was deleted, false if it did not exist.</returns>
|
||||
Task<bool> DeleteAsync(string hash, CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace StellaOps.Signals.Storage.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Context required to persist a runtime-facts artifact batch.
|
||||
/// </summary>
|
||||
public sealed record RuntimeFactsArtifactSaveRequest(
|
||||
string TenantId,
|
||||
string SubjectKey,
|
||||
string Hash,
|
||||
string ContentType,
|
||||
string FileName,
|
||||
long BatchSize,
|
||||
string? ProvenanceSource);
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace StellaOps.Signals.Storage.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Result returned after storing a runtime-facts artifact.
|
||||
/// </summary>
|
||||
public sealed record StoredRuntimeFactsArtifact(
|
||||
string Path,
|
||||
long Length,
|
||||
string Hash,
|
||||
string ContentType,
|
||||
string CasUri);
|
||||
@@ -0,0 +1,246 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Tests;
|
||||
|
||||
public class EdgeBundleIngestionServiceTests
|
||||
{
|
||||
private readonly EdgeBundleIngestionService _service;
|
||||
private const string TestTenantId = "test-tenant";
|
||||
private const string TestGraphHash = "blake3:abc123def456";
|
||||
|
||||
public EdgeBundleIngestionServiceTests()
|
||||
{
|
||||
var opts = new SignalsOptions();
|
||||
opts.Storage.RootPath = Path.GetTempPath();
|
||||
var options = Microsoft.Extensions.Options.Options.Create(opts);
|
||||
_service = new EdgeBundleIngestionService(NullLogger<EdgeBundleIngestionService>.Instance, options);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_ParsesBundleAndStoresDocument()
|
||||
{
|
||||
// Arrange
|
||||
var bundle = new
|
||||
{
|
||||
schema = "edge-bundle-v1",
|
||||
bundleId = "bundle:test123",
|
||||
graphHash = TestGraphHash,
|
||||
bundleReason = "RuntimeHits",
|
||||
generatedAt = DateTimeOffset.UtcNow.ToString("O"),
|
||||
edges = new[]
|
||||
{
|
||||
new { from = "func_a", to = "func_b", kind = "call", reason = "RuntimeHit", revoked = false, confidence = 0.9 },
|
||||
new { from = "func_b", to = "func_c", kind = "call", reason = "RuntimeHit", revoked = false, confidence = 0.8 }
|
||||
}
|
||||
};
|
||||
var bundleJson = JsonSerializer.Serialize(bundle);
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(bundleJson));
|
||||
|
||||
// Act
|
||||
var result = await _service.IngestAsync(TestTenantId, stream, null);
|
||||
|
||||
// Assert
|
||||
Assert.Equal("bundle:test123", result.BundleId);
|
||||
Assert.Equal(TestGraphHash, result.GraphHash);
|
||||
Assert.Equal("RuntimeHits", result.BundleReason);
|
||||
Assert.Equal(2, result.EdgeCount);
|
||||
Assert.Equal(0, result.RevokedCount);
|
||||
Assert.False(result.Quarantined);
|
||||
Assert.Contains("cas://reachability/edges/", result.CasUri);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_TracksRevokedEdgesForQuarantine()
|
||||
{
|
||||
// Arrange
|
||||
var bundle = new
|
||||
{
|
||||
schema = "edge-bundle-v1",
|
||||
bundleId = "bundle:revoked123",
|
||||
graphHash = TestGraphHash,
|
||||
bundleReason = "Revoked",
|
||||
edges = new[]
|
||||
{
|
||||
new { from = "func_a", to = "func_b", kind = "call", reason = "Revoked", revoked = true, confidence = 1.0 },
|
||||
new { from = "func_b", to = "func_c", kind = "call", reason = "TargetRemoved", revoked = true, confidence = 1.0 },
|
||||
new { from = "func_c", to = "func_d", kind = "call", reason = "LowConfidence", revoked = false, confidence = 0.3 }
|
||||
}
|
||||
};
|
||||
var bundleJson = JsonSerializer.Serialize(bundle);
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(bundleJson));
|
||||
|
||||
// Act
|
||||
var result = await _service.IngestAsync(TestTenantId, stream, null);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(3, result.EdgeCount);
|
||||
Assert.Equal(2, result.RevokedCount);
|
||||
Assert.True(result.Quarantined);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsEdgeRevokedAsync_ReturnsTrueForRevokedEdges()
|
||||
{
|
||||
// Arrange
|
||||
var bundle = new
|
||||
{
|
||||
bundleId = "bundle:revoke-check",
|
||||
graphHash = TestGraphHash,
|
||||
bundleReason = "Revoked",
|
||||
edges = new[]
|
||||
{
|
||||
new { from = "vuln_func", to = "patched_func", kind = "call", reason = "Revoked", revoked = true, confidence = 1.0 }
|
||||
}
|
||||
};
|
||||
var bundleJson = JsonSerializer.Serialize(bundle);
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(bundleJson));
|
||||
await _service.IngestAsync(TestTenantId, stream, null);
|
||||
|
||||
// Act & Assert
|
||||
Assert.True(await _service.IsEdgeRevokedAsync(TestTenantId, TestGraphHash, "vuln_func", "patched_func"));
|
||||
Assert.False(await _service.IsEdgeRevokedAsync(TestTenantId, TestGraphHash, "other_func", "some_func"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetBundlesForGraphAsync_ReturnsAllBundlesForGraph()
|
||||
{
|
||||
// Arrange - ingest multiple bundles
|
||||
var graphHash = $"blake3:graph_{Guid.NewGuid():N}";
|
||||
|
||||
var bundle1 = new { bundleId = "bundle:1", graphHash, bundleReason = "RuntimeHits", edges = new[] { new { from = "a", to = "b", kind = "call", reason = "RuntimeHit", revoked = false, confidence = 1.0 } } };
|
||||
var bundle2 = new { bundleId = "bundle:2", graphHash, bundleReason = "InitArray", edges = new[] { new { from = "c", to = "d", kind = "call", reason = "InitArray", revoked = false, confidence = 1.0 } } };
|
||||
|
||||
using var stream1 = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(bundle1)));
|
||||
using var stream2 = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(bundle2)));
|
||||
|
||||
await _service.IngestAsync(TestTenantId, stream1, null);
|
||||
await _service.IngestAsync(TestTenantId, stream2, null);
|
||||
|
||||
// Act
|
||||
var bundles = await _service.GetBundlesForGraphAsync(TestTenantId, graphHash);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(2, bundles.Length);
|
||||
Assert.Contains(bundles, b => b.BundleId == "bundle:1");
|
||||
Assert.Contains(bundles, b => b.BundleId == "bundle:2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetRevokedEdgesAsync_ReturnsOnlyRevokedEdges()
|
||||
{
|
||||
// Arrange
|
||||
var graphHash = $"blake3:revoked_graph_{Guid.NewGuid():N}";
|
||||
var bundle = new
|
||||
{
|
||||
bundleId = "bundle:mixed",
|
||||
graphHash,
|
||||
bundleReason = "Revoked",
|
||||
edges = new[]
|
||||
{
|
||||
new { from = "a", to = "b", kind = "call", reason = "Revoked", revoked = true, confidence = 1.0 },
|
||||
new { from = "c", to = "d", kind = "call", reason = "RuntimeHit", revoked = false, confidence = 0.9 },
|
||||
new { from = "e", to = "f", kind = "call", reason = "Revoked", revoked = true, confidence = 1.0 }
|
||||
}
|
||||
};
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(bundle)));
|
||||
await _service.IngestAsync(TestTenantId, stream, null);
|
||||
|
||||
// Act
|
||||
var revokedEdges = await _service.GetRevokedEdgesAsync(TestTenantId, graphHash);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(2, revokedEdges.Length);
|
||||
Assert.All(revokedEdges, e => Assert.True(e.Revoked));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_WithDsseStream_SetsVerifiedAndDsseFields()
|
||||
{
|
||||
// Arrange
|
||||
var bundle = new
|
||||
{
|
||||
bundleId = "bundle:verified",
|
||||
graphHash = TestGraphHash,
|
||||
bundleReason = "RuntimeHits",
|
||||
edges = new[] { new { from = "a", to = "b", kind = "call", reason = "RuntimeHit", revoked = false, confidence = 1.0 } }
|
||||
};
|
||||
var dsseEnvelope = new
|
||||
{
|
||||
payloadType = "application/vnd.stellaops.edgebundle.predicate+json",
|
||||
payload = Convert.ToBase64String(Encoding.UTF8.GetBytes("{}")),
|
||||
signatures = new[] { new { keyid = "test", sig = "abc123" } }
|
||||
};
|
||||
|
||||
using var bundleStream = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(bundle)));
|
||||
using var dsseStream = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(dsseEnvelope)));
|
||||
|
||||
// Act
|
||||
var result = await _service.IngestAsync(TestTenantId, bundleStream, dsseStream);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(result.DsseCasUri);
|
||||
Assert.EndsWith(".dsse", result.DsseCasUri);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_ThrowsOnMissingGraphHash()
|
||||
{
|
||||
// Arrange
|
||||
var bundle = new
|
||||
{
|
||||
bundleId = "bundle:no-graph",
|
||||
bundleReason = "RuntimeHits",
|
||||
edges = Array.Empty<object>()
|
||||
};
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(bundle)));
|
||||
|
||||
// Act & Assert
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => _service.IngestAsync(TestTenantId, stream, null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_UpdatesExistingBundleWithSameId()
|
||||
{
|
||||
// Arrange
|
||||
var graphHash = $"blake3:update_test_{Guid.NewGuid():N}";
|
||||
var bundle1 = new
|
||||
{
|
||||
bundleId = "bundle:same-id",
|
||||
graphHash,
|
||||
bundleReason = "RuntimeHits",
|
||||
edges = new[] { new { from = "a", to = "b", kind = "call", reason = "RuntimeHit", revoked = false, confidence = 0.5 } }
|
||||
};
|
||||
var bundle2 = new
|
||||
{
|
||||
bundleId = "bundle:same-id",
|
||||
graphHash,
|
||||
bundleReason = "RuntimeHits",
|
||||
edges = new[]
|
||||
{
|
||||
new { from = "a", to = "b", kind = "call", reason = "RuntimeHit", revoked = false, confidence = 0.9 },
|
||||
new { from = "c", to = "d", kind = "call", reason = "RuntimeHit", revoked = false, confidence = 0.8 }
|
||||
}
|
||||
};
|
||||
|
||||
using var stream1 = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(bundle1)));
|
||||
using var stream2 = new MemoryStream(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(bundle2)));
|
||||
|
||||
// Act
|
||||
await _service.IngestAsync(TestTenantId, stream1, null);
|
||||
await _service.IngestAsync(TestTenantId, stream2, null);
|
||||
|
||||
// Assert
|
||||
var bundles = await _service.GetBundlesForGraphAsync(TestTenantId, graphHash);
|
||||
Assert.Single(bundles);
|
||||
Assert.Equal(2, bundles[0].Edges.Count); // Updated to have 2 edges
|
||||
}
|
||||
}
|
||||
@@ -196,6 +196,18 @@ public class ReachabilityScoringServiceTests
|
||||
Last = document;
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<ReachabilityFactDocument>> GetExpiredAsync(DateTimeOffset olderThan, int limit, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<ReachabilityFactDocument>>(Array.Empty<ReachabilityFactDocument>());
|
||||
|
||||
public Task<bool> DeleteAsync(string id, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(true);
|
||||
|
||||
public Task<int> GetRuntimeFactsCountAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task TrimRuntimeFactsAsync(string subjectKey, int maxRuntimeFacts, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class InMemoryReachabilityCache : IReachabilityCache
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Cryptography;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Services;
|
||||
using StellaOps.Signals.Storage;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Tests;
|
||||
|
||||
public class RuntimeFactsBatchIngestionTests
|
||||
{
|
||||
private const string TestTenantId = "test-tenant";
|
||||
private const string TestCallgraphId = "test-callgraph-123";
|
||||
|
||||
[Fact]
|
||||
public async Task IngestBatchAsync_ParsesNdjsonAndStoresArtifact()
|
||||
{
|
||||
// Arrange
|
||||
var repository = new InMemoryReachabilityFactRepository();
|
||||
var artifactStore = new InMemoryRuntimeFactsArtifactStore();
|
||||
var cryptoHash = DefaultCryptoHash.CreateForTests();
|
||||
var service = CreateService(repository, artifactStore, cryptoHash);
|
||||
|
||||
var events = new[]
|
||||
{
|
||||
new { symbolId = "func_a", hitCount = 5, callgraphId = TestCallgraphId, subject = new { scanId = "scan-1" } },
|
||||
new { symbolId = "func_b", hitCount = 3, callgraphId = TestCallgraphId, subject = new { scanId = "scan-1" } },
|
||||
new { symbolId = "func_c", hitCount = 1, callgraphId = TestCallgraphId, subject = new { scanId = "scan-1" } }
|
||||
};
|
||||
var ndjson = string.Join("\n", events.Select(e => JsonSerializer.Serialize(e)));
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ndjson));
|
||||
|
||||
// Act
|
||||
var result = await service.IngestBatchAsync(TestTenantId, stream, "application/x-ndjson", CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(result);
|
||||
Assert.StartsWith("cas://reachability/runtime-facts/", result.CasUri);
|
||||
Assert.StartsWith("blake3:", result.BatchHash);
|
||||
Assert.Equal(1, result.ProcessedCount);
|
||||
Assert.Equal(3, result.TotalEvents);
|
||||
Assert.Equal(9, result.TotalHitCount);
|
||||
Assert.Contains("scan-1", result.SubjectKeys);
|
||||
Assert.True(artifactStore.StoredArtifacts.Count > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestBatchAsync_HandlesGzipCompressedContent()
|
||||
{
|
||||
// Arrange
|
||||
var repository = new InMemoryReachabilityFactRepository();
|
||||
var artifactStore = new InMemoryRuntimeFactsArtifactStore();
|
||||
var cryptoHash = DefaultCryptoHash.CreateForTests();
|
||||
var service = CreateService(repository, artifactStore, cryptoHash);
|
||||
|
||||
var events = new[]
|
||||
{
|
||||
new { symbolId = "func_gzip", hitCount = 10, callgraphId = TestCallgraphId, subject = new { scanId = "scan-gzip" } }
|
||||
};
|
||||
var ndjson = string.Join("\n", events.Select(e => JsonSerializer.Serialize(e)));
|
||||
|
||||
using var compressedStream = new MemoryStream();
|
||||
await using (var gzipStream = new GZipStream(compressedStream, CompressionMode.Compress, leaveOpen: true))
|
||||
{
|
||||
await gzipStream.WriteAsync(Encoding.UTF8.GetBytes(ndjson));
|
||||
}
|
||||
|
||||
compressedStream.Position = 0;
|
||||
|
||||
// Act
|
||||
var result = await service.IngestBatchAsync(TestTenantId, compressedStream, "application/gzip", CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(1, result.ProcessedCount);
|
||||
Assert.Equal(1, result.TotalEvents);
|
||||
Assert.Equal(10, result.TotalHitCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestBatchAsync_GroupsEventsBySubject()
|
||||
{
|
||||
// Arrange
|
||||
var repository = new InMemoryReachabilityFactRepository();
|
||||
var artifactStore = new InMemoryRuntimeFactsArtifactStore();
|
||||
var cryptoHash = DefaultCryptoHash.CreateForTests();
|
||||
var service = CreateService(repository, artifactStore, cryptoHash);
|
||||
|
||||
var events = new[]
|
||||
{
|
||||
new { symbolId = "func_a", hitCount = 1, callgraphId = "cg-1", subject = new { scanId = "scan-1" } },
|
||||
new { symbolId = "func_b", hitCount = 2, callgraphId = "cg-1", subject = new { scanId = "scan-1" } },
|
||||
new { symbolId = "func_c", hitCount = 3, callgraphId = "cg-2", subject = new { scanId = "scan-2" } },
|
||||
new { symbolId = "func_d", hitCount = 4, callgraphId = "cg-2", subject = new { scanId = "scan-2" } }
|
||||
};
|
||||
var ndjson = string.Join("\n", events.Select(e => JsonSerializer.Serialize(e)));
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ndjson));
|
||||
|
||||
// Act
|
||||
var result = await service.IngestBatchAsync(TestTenantId, stream, "application/x-ndjson", CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(2, result.ProcessedCount);
|
||||
Assert.Equal(4, result.TotalEvents);
|
||||
Assert.Equal(10, result.TotalHitCount);
|
||||
Assert.Contains("scan-1", result.SubjectKeys);
|
||||
Assert.Contains("scan-2", result.SubjectKeys);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestBatchAsync_LinksCasUriToFactDocument()
|
||||
{
|
||||
// Arrange
|
||||
var repository = new InMemoryReachabilityFactRepository();
|
||||
var artifactStore = new InMemoryRuntimeFactsArtifactStore();
|
||||
var cryptoHash = DefaultCryptoHash.CreateForTests();
|
||||
var service = CreateService(repository, artifactStore, cryptoHash);
|
||||
|
||||
var events = new[]
|
||||
{
|
||||
new { symbolId = "func_link", hitCount = 1, callgraphId = TestCallgraphId, subject = new { scanId = "scan-link" } }
|
||||
};
|
||||
var ndjson = string.Join("\n", events.Select(e => JsonSerializer.Serialize(e)));
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ndjson));
|
||||
|
||||
// Act
|
||||
var result = await service.IngestBatchAsync(TestTenantId, stream, "application/x-ndjson", CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
var fact = await repository.GetBySubjectAsync("scan-link", CancellationToken.None);
|
||||
Assert.NotNull(fact);
|
||||
Assert.Equal(result.CasUri, fact.RuntimeFactsBatchUri);
|
||||
Assert.Equal(result.BatchHash, fact.RuntimeFactsBatchHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestBatchAsync_SkipsInvalidLines()
|
||||
{
|
||||
// Arrange
|
||||
var repository = new InMemoryReachabilityFactRepository();
|
||||
var artifactStore = new InMemoryRuntimeFactsArtifactStore();
|
||||
var cryptoHash = DefaultCryptoHash.CreateForTests();
|
||||
var service = CreateService(repository, artifactStore, cryptoHash);
|
||||
|
||||
var ndjson = """
|
||||
{"symbolId": "func_valid", "hitCount": 1, "callgraphId": "cg-1", "subject": {"scanId": "scan-skip"}}
|
||||
invalid json line
|
||||
{"symbolId": "", "hitCount": 1, "callgraphId": "cg-1", "subject": {"scanId": "scan-skip"}}
|
||||
{"symbolId": "func_valid2", "hitCount": 2, "callgraphId": "cg-1", "subject": {"scanId": "scan-skip"}}
|
||||
""";
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ndjson));
|
||||
|
||||
// Act
|
||||
var result = await service.IngestBatchAsync(TestTenantId, stream, "application/x-ndjson", CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(1, result.ProcessedCount);
|
||||
Assert.Equal(2, result.TotalEvents); // Only valid lines
|
||||
Assert.Equal(3, result.TotalHitCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestBatchAsync_WorksWithoutArtifactStore()
|
||||
{
|
||||
// Arrange
|
||||
var repository = new InMemoryReachabilityFactRepository();
|
||||
var service = CreateService(repository, artifactStore: null, cryptoHash: null);
|
||||
|
||||
var events = new[]
|
||||
{
|
||||
new { symbolId = "func_no_cas", hitCount = 5, callgraphId = TestCallgraphId, subject = new { scanId = "scan-no-cas" } }
|
||||
};
|
||||
var ndjson = string.Join("\n", events.Select(e => JsonSerializer.Serialize(e)));
|
||||
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ndjson));
|
||||
|
||||
// Act
|
||||
var result = await service.IngestBatchAsync(TestTenantId, stream, "application/x-ndjson", CancellationToken.None);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(result);
|
||||
Assert.StartsWith("cas://reachability/runtime-facts/", result.CasUri);
|
||||
Assert.Equal(1, result.ProcessedCount);
|
||||
}
|
||||
|
||||
private static RuntimeFactsIngestionService CreateService(
|
||||
IReachabilityFactRepository repository,
|
||||
IRuntimeFactsArtifactStore? artifactStore,
|
||||
ICryptoHash? cryptoHash)
|
||||
{
|
||||
var cache = new InMemoryReachabilityCache();
|
||||
var eventsPublisher = new NullEventsPublisher();
|
||||
var scoringService = new StubReachabilityScoringService();
|
||||
var provenanceNormalizer = new StubProvenanceNormalizer();
|
||||
|
||||
return new RuntimeFactsIngestionService(
|
||||
repository,
|
||||
TimeProvider.System,
|
||||
cache,
|
||||
eventsPublisher,
|
||||
scoringService,
|
||||
provenanceNormalizer,
|
||||
NullLogger<RuntimeFactsIngestionService>.Instance,
|
||||
artifactStore,
|
||||
cryptoHash);
|
||||
}
|
||||
|
||||
private sealed class InMemoryReachabilityFactRepository : IReachabilityFactRepository
|
||||
{
|
||||
private readonly Dictionary<string, ReachabilityFactDocument> _facts = new(StringComparer.Ordinal);
|
||||
|
||||
public Task<ReachabilityFactDocument> UpsertAsync(ReachabilityFactDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
_facts[document.SubjectKey] = document;
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
|
||||
public Task<ReachabilityFactDocument?> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(_facts.TryGetValue(subjectKey, out var doc) ? doc : null);
|
||||
|
||||
public Task<IReadOnlyList<ReachabilityFactDocument>> GetExpiredAsync(DateTimeOffset cutoff, int limit, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<ReachabilityFactDocument>>([]);
|
||||
|
||||
public Task<bool> DeleteAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
var removed = _facts.Remove(subjectKey);
|
||||
return Task.FromResult(removed);
|
||||
}
|
||||
|
||||
public Task<int> GetRuntimeFactsCountAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(_facts.TryGetValue(subjectKey, out var doc) ? doc.RuntimeFacts?.Count ?? 0 : 0);
|
||||
|
||||
public Task TrimRuntimeFactsAsync(string subjectKey, int maxCount, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class InMemoryRuntimeFactsArtifactStore : IRuntimeFactsArtifactStore
|
||||
{
|
||||
public Dictionary<string, StoredRuntimeFactsArtifact> StoredArtifacts { get; } = new(StringComparer.Ordinal);
|
||||
|
||||
public async Task<StoredRuntimeFactsArtifact> SaveAsync(RuntimeFactsArtifactSaveRequest request, Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
using var ms = new MemoryStream();
|
||||
await content.CopyToAsync(ms, cancellationToken);
|
||||
|
||||
var artifact = new StoredRuntimeFactsArtifact(
|
||||
Path: $"cas/reachability/runtime-facts/{request.Hash[..2]}/{request.Hash}/{request.FileName}",
|
||||
Length: ms.Length,
|
||||
Hash: request.Hash,
|
||||
ContentType: request.ContentType,
|
||||
CasUri: $"cas://reachability/runtime-facts/{request.Hash}");
|
||||
|
||||
StoredArtifacts[request.Hash] = artifact;
|
||||
return artifact;
|
||||
}
|
||||
|
||||
public Task<Stream?> GetAsync(string hash, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<Stream?>(null);
|
||||
|
||||
public Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(StoredArtifacts.ContainsKey(hash));
|
||||
|
||||
public Task<bool> DeleteAsync(string hash, CancellationToken cancellationToken)
|
||||
{
|
||||
return Task.FromResult(StoredArtifacts.Remove(hash));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryReachabilityCache : IReachabilityCache
|
||||
{
|
||||
public Task<ReachabilityFactDocument?> GetAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<ReachabilityFactDocument?>(null);
|
||||
|
||||
public Task SetAsync(ReachabilityFactDocument document, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task InvalidateAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class NullEventsPublisher : IEventsPublisher
|
||||
{
|
||||
public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class StubReachabilityScoringService : IReachabilityScoringService
|
||||
{
|
||||
public Task<ReachabilityFactDocument> RecomputeAsync(ReachabilityRecomputeRequest request, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(new ReachabilityFactDocument { SubjectKey = request.Subject.ToSubjectKey() });
|
||||
}
|
||||
|
||||
private sealed class StubProvenanceNormalizer : IRuntimeFactsProvenanceNormalizer
|
||||
{
|
||||
public ContextFacts CreateContextFacts(
|
||||
IEnumerable<RuntimeFactEvent> events,
|
||||
ReachabilitySubject subject,
|
||||
string callgraphId,
|
||||
Dictionary<string, string?>? metadata,
|
||||
DateTimeOffset timestamp)
|
||||
=> new();
|
||||
|
||||
public ProvenanceFeed NormalizeToFeed(
|
||||
IEnumerable<RuntimeFactEvent> events,
|
||||
ReachabilitySubject subject,
|
||||
string callgraphId,
|
||||
Dictionary<string, string?>? metadata,
|
||||
DateTimeOffset generatedAt)
|
||||
=> new() { FeedId = "test-feed", GeneratedAt = generatedAt };
|
||||
}
|
||||
}
|
||||
@@ -96,6 +96,18 @@ public class RuntimeFactsIngestionServiceTests
|
||||
Last = document;
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<ReachabilityFactDocument>> GetExpiredAsync(DateTimeOffset olderThan, int limit, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<ReachabilityFactDocument>>(Array.Empty<ReachabilityFactDocument>());
|
||||
|
||||
public Task<bool> DeleteAsync(string id, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(true);
|
||||
|
||||
public Task<int> GetRuntimeFactsCountAsync(string subjectKey, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(0);
|
||||
|
||||
public Task TrimRuntimeFactsAsync(string subjectKey, int maxRuntimeFacts, CancellationToken cancellationToken)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
private sealed class InMemoryReachabilityCache : IReachabilityCache
|
||||
|
||||
Reference in New Issue
Block a user