feat: Implement Runtime Facts ingestion service and NDJSON reader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added RuntimeFactsNdjsonReader for reading NDJSON formatted runtime facts. - Introduced IRuntimeFactsIngestionService interface and its implementation. - Enhanced Program.cs to register new services and endpoints for runtime facts. - Updated CallgraphIngestionService to include CAS URI in stored artifacts. - Created RuntimeFactsValidationException for validation errors during ingestion. - Added tests for RuntimeFactsIngestionService and RuntimeFactsNdjsonReader. - Implemented SignalsSealedModeMonitor for compliance checks in sealed mode. - Updated project dependencies for testing utilities.
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Reachability.Tests;
|
||||
|
||||
public sealed class RuntimeFactsIngestionServiceTests
|
||||
{
|
||||
private readonly FakeReachabilityFactRepository repository = new();
|
||||
private readonly FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2025-11-09T10:15:00Z", null, System.Globalization.DateTimeStyles.AssumeUniversal));
|
||||
private readonly RuntimeFactsIngestionService sut;
|
||||
|
||||
public RuntimeFactsIngestionServiceTests()
|
||||
{
|
||||
sut = new RuntimeFactsIngestionService(repository, timeProvider, NullLogger<RuntimeFactsIngestionService>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_InsertsAggregatedFacts()
|
||||
{
|
||||
var request = new RuntimeFactsIngestRequest
|
||||
{
|
||||
CallgraphId = "cg-123",
|
||||
Subject = new ReachabilitySubject { ScanId = "scan-1" },
|
||||
Events = new List<RuntimeFactEvent>
|
||||
{
|
||||
new()
|
||||
{
|
||||
SymbolId = "symbol::foo",
|
||||
HitCount = 3,
|
||||
Metadata = new Dictionary<string, string?> { ["thread"] = "main" }
|
||||
},
|
||||
new()
|
||||
{
|
||||
SymbolId = "symbol::foo",
|
||||
HitCount = 2
|
||||
},
|
||||
new()
|
||||
{
|
||||
SymbolId = "symbol::bar",
|
||||
CodeId = "elf:abcd",
|
||||
LoaderBase = "0x4000",
|
||||
HitCount = 1
|
||||
}
|
||||
},
|
||||
Metadata = new Dictionary<string, string?> { ["source"] = "zastava" }
|
||||
};
|
||||
|
||||
var response = await sut.IngestAsync(request, CancellationToken.None);
|
||||
|
||||
response.SubjectKey.Should().Be("scan-1");
|
||||
response.CallgraphId.Should().Be("cg-123");
|
||||
response.RuntimeFactCount.Should().Be(2);
|
||||
response.TotalHitCount.Should().Be(6);
|
||||
response.StoredAt.Should().Be(timeProvider.GetUtcNow());
|
||||
|
||||
repository.LastUpsert.Should().NotBeNull();
|
||||
repository.LastUpsert!.RuntimeFacts.Should().NotBeNull();
|
||||
repository.LastUpsert!.RuntimeFacts.Should().HaveCount(2);
|
||||
repository.LastUpsert!.RuntimeFacts![0].SymbolId.Should().Be("symbol::bar");
|
||||
repository.LastUpsert!.RuntimeFacts![0].HitCount.Should().Be(1);
|
||||
repository.LastUpsert!.RuntimeFacts![1].SymbolId.Should().Be("symbol::foo");
|
||||
repository.LastUpsert!.RuntimeFacts![1].HitCount.Should().Be(5);
|
||||
repository.LastUpsert!.Metadata.Should().ContainKey("source");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAsync_MergesExistingDocument()
|
||||
{
|
||||
var existing = new ReachabilityFactDocument
|
||||
{
|
||||
Id = "507f1f77bcf86cd799439011",
|
||||
Subject = new ReachabilitySubject { ImageDigest = "sha256:abc" },
|
||||
SubjectKey = "sha256:abc",
|
||||
CallgraphId = "cg-old",
|
||||
RuntimeFacts = new List<RuntimeFactDocument>
|
||||
{
|
||||
new() { SymbolId = "old::symbol", HitCount = 1, Metadata = new Dictionary<string, string?> { ["thread"] = "bg" } }
|
||||
}
|
||||
};
|
||||
|
||||
repository.LastUpsert = existing;
|
||||
|
||||
var request = new RuntimeFactsIngestRequest
|
||||
{
|
||||
Subject = new ReachabilitySubject { ImageDigest = "sha256:abc" },
|
||||
CallgraphId = "cg-new",
|
||||
Events = new List<RuntimeFactEvent>
|
||||
{
|
||||
new() { SymbolId = "new::symbol", HitCount = 2 },
|
||||
new() { SymbolId = "old::symbol", HitCount = 3, Metadata = new Dictionary<string, string?> { ["thread"] = "main" } }
|
||||
}
|
||||
};
|
||||
|
||||
var response = await sut.IngestAsync(request, CancellationToken.None);
|
||||
|
||||
response.FactId.Should().Be(existing.Id);
|
||||
repository.LastUpsert!.RuntimeFacts.Should().HaveCount(2);
|
||||
repository.LastUpsert!.RuntimeFacts![0].SymbolId.Should().Be("new::symbol");
|
||||
repository.LastUpsert!.RuntimeFacts![1].SymbolId.Should().Be("old::symbol");
|
||||
repository.LastUpsert!.RuntimeFacts![1].HitCount.Should().Be(4);
|
||||
repository.LastUpsert!.RuntimeFacts![1].Metadata.Should().ContainKey("thread").WhoseValue.Should().Be("main");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
public async Task IngestAsync_ValidatesCallgraphId(string? callgraphId)
|
||||
{
|
||||
var request = new RuntimeFactsIngestRequest
|
||||
{
|
||||
Subject = new ReachabilitySubject { ScanId = "scan" },
|
||||
CallgraphId = callgraphId ?? string.Empty,
|
||||
Events = new List<RuntimeFactEvent> { new() { SymbolId = "foo" } }
|
||||
};
|
||||
|
||||
await Assert.ThrowsAsync<RuntimeFactsValidationException>(() => sut.IngestAsync(request, CancellationToken.None));
|
||||
}
|
||||
|
||||
private sealed class FakeReachabilityFactRepository : IReachabilityFactRepository
|
||||
{
|
||||
public ReachabilityFactDocument? LastUpsert { get; set; }
|
||||
|
||||
public Task<ReachabilityFactDocument> UpsertAsync(ReachabilityFactDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
LastUpsert = document;
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
|
||||
public Task<ReachabilityFactDocument?> GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(LastUpsert is { SubjectKey: not null } doc && doc.SubjectKey == subjectKey ? doc : null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Signals.Parsing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Reachability.Tests;
|
||||
|
||||
public sealed class RuntimeFactsNdjsonReaderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task ReadAsync_ParsesLines()
|
||||
{
|
||||
var ndjson = """
|
||||
{"symbolId":"sym::foo","hitCount":2}
|
||||
{"symbolId":"sym::bar","codeId":"elf:abcd","loaderBase":"0x1000","metadata":{"thread":"bg"}}
|
||||
|
||||
""";
|
||||
|
||||
await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ndjson));
|
||||
var events = await RuntimeFactsNdjsonReader.ReadAsync(stream, gzipEncoded: false, CancellationToken.None);
|
||||
|
||||
events.Should().HaveCount(2);
|
||||
events[0].SymbolId.Should().Be("sym::foo");
|
||||
events[1].LoaderBase.Should().Be("0x1000");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadAsync_HandlesGzip()
|
||||
{
|
||||
var ndjson = """
|
||||
{"symbolId":"sym::foo"}
|
||||
""";
|
||||
await using var compressed = new MemoryStream();
|
||||
await using (var gzip = new GZipStream(compressed, CompressionLevel.Optimal, leaveOpen: true))
|
||||
await using (var writer = new StreamWriter(gzip, Encoding.UTF8, leaveOpen: true))
|
||||
{
|
||||
await writer.WriteAsync(ndjson);
|
||||
}
|
||||
|
||||
compressed.Position = 0;
|
||||
|
||||
var events = await RuntimeFactsNdjsonReader.ReadAsync(compressed, gzipEncoded: true, CancellationToken.None);
|
||||
events.Should().HaveCount(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using StellaOps.Signals.Hosting;
|
||||
using StellaOps.Signals.Options;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Signals.Reachability.Tests;
|
||||
|
||||
public sealed class SignalsSealedModeMonitorTests : IDisposable
|
||||
{
|
||||
private readonly string tempDir = Path.Combine(Path.GetTempPath(), $"signals-sealed-tests-{Guid.NewGuid():N}");
|
||||
|
||||
[Fact]
|
||||
public void IsCompliant_WhenEnforcementDisabled_ReturnsTrue()
|
||||
{
|
||||
var options = new SignalsOptions();
|
||||
options.AirGap.SealedMode.EnforcementEnabled = false;
|
||||
|
||||
var monitor = CreateMonitor(options);
|
||||
|
||||
monitor.IsCompliant(out _).Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsCompliant_WhenEvidenceMissing_ReturnsFalse()
|
||||
{
|
||||
var options = CreateEnforcedOptions();
|
||||
options.AirGap.SealedMode.EvidencePath = Path.Combine(tempDir, "missing.json");
|
||||
|
||||
var monitor = CreateMonitor(options);
|
||||
|
||||
monitor.IsCompliant(out var reason).Should().BeFalse();
|
||||
reason.Should().Contain("not found");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsCompliant_WhenEvidenceFresh_ReturnsTrue()
|
||||
{
|
||||
var evidencePath = CreateEvidenceFile(TimeSpan.Zero);
|
||||
var options = CreateEnforcedOptions();
|
||||
options.AirGap.SealedMode.EvidencePath = evidencePath;
|
||||
|
||||
var monitor = CreateMonitor(options);
|
||||
|
||||
monitor.IsCompliant(out _).Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IsCompliant_WhenEvidenceStale_ReturnsFalse()
|
||||
{
|
||||
var evidencePath = CreateEvidenceFile(TimeSpan.FromHours(7));
|
||||
var options = CreateEnforcedOptions();
|
||||
options.AirGap.SealedMode.EvidencePath = evidencePath;
|
||||
|
||||
var monitor = CreateMonitor(options);
|
||||
|
||||
monitor.IsCompliant(out _).Should().BeFalse();
|
||||
}
|
||||
|
||||
private SignalsOptions CreateEnforcedOptions()
|
||||
{
|
||||
var options = new SignalsOptions();
|
||||
options.AirGap.SealedMode.EnforcementEnabled = true;
|
||||
options.AirGap.SealedMode.MaxEvidenceAge = TimeSpan.FromHours(6);
|
||||
options.AirGap.SealedMode.CacheLifetime = TimeSpan.FromSeconds(1);
|
||||
return options;
|
||||
}
|
||||
|
||||
private string CreateEvidenceFile(TimeSpan age)
|
||||
{
|
||||
Directory.CreateDirectory(tempDir);
|
||||
var path = Path.Combine(tempDir, $"{Guid.NewGuid():N}.json");
|
||||
File.WriteAllText(path, "{}");
|
||||
if (age > TimeSpan.Zero)
|
||||
{
|
||||
File.SetLastWriteTimeUtc(path, DateTime.UtcNow - age);
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
private SignalsSealedModeMonitor CreateMonitor(SignalsOptions options)
|
||||
{
|
||||
return new SignalsSealedModeMonitor(
|
||||
options,
|
||||
new FakeTimeProvider(DateTimeOffset.UtcNow),
|
||||
NullLogger<SignalsSealedModeMonitor>.Instance);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(tempDir))
|
||||
{
|
||||
Directory.Delete(tempDir, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" Version="8.6.0" />
|
||||
<PackageReference Include="xunit" Version="2.7.0" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.8">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
|
||||
Reference in New Issue
Block a user