feat: Implement Runtime Facts ingestion service and NDJSON reader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added RuntimeFactsNdjsonReader for reading NDJSON formatted runtime facts.
- Introduced IRuntimeFactsIngestionService interface and its implementation.
- Enhanced Program.cs to register new services and endpoints for runtime facts.
- Updated CallgraphIngestionService to include CAS URI in stored artifacts.
- Created RuntimeFactsValidationException for validation errors during ingestion.
- Added tests for RuntimeFactsIngestionService and RuntimeFactsNdjsonReader.
- Implemented SignalsSealedModeMonitor for compliance checks in sealed mode.
- Updated project dependencies for testing utilities.
This commit is contained in:
master
2025-11-10 07:56:15 +02:00
parent 9df52d84aa
commit 69c59defdc
132 changed files with 19718 additions and 9334 deletions

View File

@@ -0,0 +1,141 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Signals.Models;
using StellaOps.Signals.Persistence;
using StellaOps.Signals.Services;
using Xunit;
namespace StellaOps.Signals.Reachability.Tests;
public sealed class RuntimeFactsIngestionServiceTests
{
private readonly FakeReachabilityFactRepository repository = new();
private readonly FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2025-11-09T10:15:00Z", null, System.Globalization.DateTimeStyles.AssumeUniversal));
private readonly RuntimeFactsIngestionService sut;
public RuntimeFactsIngestionServiceTests()
{
sut = new RuntimeFactsIngestionService(repository, timeProvider, NullLogger<RuntimeFactsIngestionService>.Instance);
}
[Fact]
public async Task IngestAsync_InsertsAggregatedFacts()
{
var request = new RuntimeFactsIngestRequest
{
CallgraphId = "cg-123",
Subject = new ReachabilitySubject { ScanId = "scan-1" },
Events = new List<RuntimeFactEvent>
{
new()
{
SymbolId = "symbol::foo",
HitCount = 3,
Metadata = new Dictionary<string, string?> { ["thread"] = "main" }
},
new()
{
SymbolId = "symbol::foo",
HitCount = 2
},
new()
{
SymbolId = "symbol::bar",
CodeId = "elf:abcd",
LoaderBase = "0x4000",
HitCount = 1
}
},
Metadata = new Dictionary<string, string?> { ["source"] = "zastava" }
};
var response = await sut.IngestAsync(request, CancellationToken.None);
response.SubjectKey.Should().Be("scan-1");
response.CallgraphId.Should().Be("cg-123");
response.RuntimeFactCount.Should().Be(2);
response.TotalHitCount.Should().Be(6);
response.StoredAt.Should().Be(timeProvider.GetUtcNow());
repository.LastUpsert.Should().NotBeNull();
repository.LastUpsert!.RuntimeFacts.Should().NotBeNull();
repository.LastUpsert!.RuntimeFacts.Should().HaveCount(2);
repository.LastUpsert!.RuntimeFacts![0].SymbolId.Should().Be("symbol::bar");
repository.LastUpsert!.RuntimeFacts![0].HitCount.Should().Be(1);
repository.LastUpsert!.RuntimeFacts![1].SymbolId.Should().Be("symbol::foo");
repository.LastUpsert!.RuntimeFacts![1].HitCount.Should().Be(5);
repository.LastUpsert!.Metadata.Should().ContainKey("source");
}
[Fact]
public async Task IngestAsync_MergesExistingDocument()
{
var existing = new ReachabilityFactDocument
{
Id = "507f1f77bcf86cd799439011",
Subject = new ReachabilitySubject { ImageDigest = "sha256:abc" },
SubjectKey = "sha256:abc",
CallgraphId = "cg-old",
RuntimeFacts = new List<RuntimeFactDocument>
{
new() { SymbolId = "old::symbol", HitCount = 1, Metadata = new Dictionary<string, string?> { ["thread"] = "bg" } }
}
};
repository.LastUpsert = existing;
var request = new RuntimeFactsIngestRequest
{
Subject = new ReachabilitySubject { ImageDigest = "sha256:abc" },
CallgraphId = "cg-new",
Events = new List<RuntimeFactEvent>
{
new() { SymbolId = "new::symbol", HitCount = 2 },
new() { SymbolId = "old::symbol", HitCount = 3, Metadata = new Dictionary<string, string?> { ["thread"] = "main" } }
}
};
var response = await sut.IngestAsync(request, CancellationToken.None);
response.FactId.Should().Be(existing.Id);
repository.LastUpsert!.RuntimeFacts.Should().HaveCount(2);
repository.LastUpsert!.RuntimeFacts![0].SymbolId.Should().Be("new::symbol");
repository.LastUpsert!.RuntimeFacts![1].SymbolId.Should().Be("old::symbol");
repository.LastUpsert!.RuntimeFacts![1].HitCount.Should().Be(4);
repository.LastUpsert!.RuntimeFacts![1].Metadata.Should().ContainKey("thread").WhoseValue.Should().Be("main");
}
[Theory]
[InlineData(null)]
[InlineData("")]
public async Task IngestAsync_ValidatesCallgraphId(string? callgraphId)
{
var request = new RuntimeFactsIngestRequest
{
Subject = new ReachabilitySubject { ScanId = "scan" },
CallgraphId = callgraphId ?? string.Empty,
Events = new List<RuntimeFactEvent> { new() { SymbolId = "foo" } }
};
await Assert.ThrowsAsync<RuntimeFactsValidationException>(() => sut.IngestAsync(request, CancellationToken.None));
}
private sealed class FakeReachabilityFactRepository : IReachabilityFactRepository
{
public ReachabilityFactDocument? LastUpsert { get; set; }
public Task<ReachabilityFactDocument> UpsertAsync(ReachabilityFactDocument document, CancellationToken cancellationToken)
{
LastUpsert = document;
return Task.FromResult(document);
}
public Task<ReachabilityFactDocument?> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken) =>
Task.FromResult(LastUpsert is { SubjectKey: not null } doc && doc.SubjectKey == subjectKey ? doc : null);
}
}

View File

@@ -0,0 +1,49 @@
using System.IO;
using System.IO.Compression;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using StellaOps.Signals.Parsing;
using Xunit;
namespace StellaOps.Signals.Reachability.Tests;
public sealed class RuntimeFactsNdjsonReaderTests
{
[Fact]
public async Task ReadAsync_ParsesLines()
{
var ndjson = """
{"symbolId":"sym::foo","hitCount":2}
{"symbolId":"sym::bar","codeId":"elf:abcd","loaderBase":"0x1000","metadata":{"thread":"bg"}}
""";
await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ndjson));
var events = await RuntimeFactsNdjsonReader.ReadAsync(stream, gzipEncoded: false, CancellationToken.None);
events.Should().HaveCount(2);
events[0].SymbolId.Should().Be("sym::foo");
events[1].LoaderBase.Should().Be("0x1000");
}
[Fact]
public async Task ReadAsync_HandlesGzip()
{
var ndjson = """
{"symbolId":"sym::foo"}
""";
await using var compressed = new MemoryStream();
await using (var gzip = new GZipStream(compressed, CompressionLevel.Optimal, leaveOpen: true))
await using (var writer = new StreamWriter(gzip, Encoding.UTF8, leaveOpen: true))
{
await writer.WriteAsync(ndjson);
}
compressed.Position = 0;
var events = await RuntimeFactsNdjsonReader.ReadAsync(compressed, gzipEncoded: true, CancellationToken.None);
events.Should().HaveCount(1);
}
}

View File

@@ -0,0 +1,100 @@
using System;
using System.IO;
using FluentAssertions;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Signals.Hosting;
using StellaOps.Signals.Options;
using Xunit;
namespace StellaOps.Signals.Reachability.Tests;
public sealed class SignalsSealedModeMonitorTests : IDisposable
{
private readonly string tempDir = Path.Combine(Path.GetTempPath(), $"signals-sealed-tests-{Guid.NewGuid():N}");
[Fact]
public void IsCompliant_WhenEnforcementDisabled_ReturnsTrue()
{
var options = new SignalsOptions();
options.AirGap.SealedMode.EnforcementEnabled = false;
var monitor = CreateMonitor(options);
monitor.IsCompliant(out _).Should().BeTrue();
}
[Fact]
public void IsCompliant_WhenEvidenceMissing_ReturnsFalse()
{
var options = CreateEnforcedOptions();
options.AirGap.SealedMode.EvidencePath = Path.Combine(tempDir, "missing.json");
var monitor = CreateMonitor(options);
monitor.IsCompliant(out var reason).Should().BeFalse();
reason.Should().Contain("not found");
}
[Fact]
public void IsCompliant_WhenEvidenceFresh_ReturnsTrue()
{
var evidencePath = CreateEvidenceFile(TimeSpan.Zero);
var options = CreateEnforcedOptions();
options.AirGap.SealedMode.EvidencePath = evidencePath;
var monitor = CreateMonitor(options);
monitor.IsCompliant(out _).Should().BeTrue();
}
[Fact]
public void IsCompliant_WhenEvidenceStale_ReturnsFalse()
{
var evidencePath = CreateEvidenceFile(TimeSpan.FromHours(7));
var options = CreateEnforcedOptions();
options.AirGap.SealedMode.EvidencePath = evidencePath;
var monitor = CreateMonitor(options);
monitor.IsCompliant(out _).Should().BeFalse();
}
private SignalsOptions CreateEnforcedOptions()
{
var options = new SignalsOptions();
options.AirGap.SealedMode.EnforcementEnabled = true;
options.AirGap.SealedMode.MaxEvidenceAge = TimeSpan.FromHours(6);
options.AirGap.SealedMode.CacheLifetime = TimeSpan.FromSeconds(1);
return options;
}
private string CreateEvidenceFile(TimeSpan age)
{
Directory.CreateDirectory(tempDir);
var path = Path.Combine(tempDir, $"{Guid.NewGuid():N}.json");
File.WriteAllText(path, "{}");
if (age > TimeSpan.Zero)
{
File.SetLastWriteTimeUtc(path, DateTime.UtcNow - age);
}
return path;
}
private SignalsSealedModeMonitor CreateMonitor(SignalsOptions options)
{
return new SignalsSealedModeMonitor(
options,
new FakeTimeProvider(DateTimeOffset.UtcNow),
NullLogger<SignalsSealedModeMonitor>.Instance);
}
public void Dispose()
{
if (Directory.Exists(tempDir))
{
Directory.Delete(tempDir, recursive: true);
}
}
}

View File

@@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
<PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" Version="8.6.0" />
<PackageReference Include="xunit" Version="2.7.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.8">
<PrivateAssets>all</PrivateAssets>