using System; using System.Collections.Generic; using System.IO; using System.Net; using System.Net.Http.Json; using System.Text; using System.Text.Json; using CycloneDX.Json; using CycloneDX.Models; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using StellaOps.Scanner.Storage.Catalog; using StellaOps.Scanner.Storage.ObjectStore; using StellaOps.Scanner.Storage.Repositories; using StellaOps.Scanner.WebService.Contracts; using StellaOps.Zastava.Core.Contracts; using StellaOps.TestKit; namespace StellaOps.Scanner.WebService.Tests; public sealed class RuntimeReconciliationTests { private const string TestImageDigest = "sha256:abc123def456"; private const string TestTenant = "tenant-alpha"; private const string TestNode = "node-a"; [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconcileEndpoint_WithNoRuntimeEvents_ReturnsNotFound() { using var factory = new ScannerApplicationFactory(); using var client = factory.CreateClient(); var request = new RuntimeReconcileRequestDto { ImageDigest = TestImageDigest }; var response = await client.PostAsJsonAsync("/api/v1/runtime/reconcile", request); Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); var payload = await response.Content.ReadFromJsonAsync(); Assert.NotNull(payload); Assert.Equal("NO_RUNTIME_EVENTS", payload!.ErrorCode); Assert.Contains("No runtime events found", payload.ErrorMessage); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconcileEndpoint_WithRuntimeEventsButNoSbom_ReturnsNoSbomError() { var mockObjectStore = new InMemoryArtifactObjectStore(); using var factory = new ScannerApplicationFactory().WithOverrides( configureServices: services => { services.RemoveAll(); services.AddSingleton(mockObjectStore); }); using var client = factory.CreateClient(); // Ingest runtime event with loaded libraries var ingestRequest = new RuntimeEventsIngestRequestDto { Events = new[] { CreateEnvelopeWithLibraries("evt-001", TestImageDigest, new[] { new RuntimeLoadedLibrary { Path = "/lib/libssl.so.3", Sha256 = "sha256:lib1hash", Inode = 1001 }, new RuntimeLoadedLibrary { Path = "/lib/libcrypto.so.3", Sha256 = "sha256:lib2hash", Inode = 1002 } }) } }; var ingestResponse = await client.PostAsJsonAsync("/api/v1/runtime/events", ingestRequest); Assert.Equal(HttpStatusCode.Accepted, ingestResponse.StatusCode); // Request reconciliation - no SBOM linked var reconcileRequest = new RuntimeReconcileRequestDto { ImageDigest = TestImageDigest }; var response = await client.PostAsJsonAsync("/api/v1/runtime/reconcile", reconcileRequest); Assert.Equal(HttpStatusCode.OK, response.StatusCode); var payload = await response.Content.ReadFromJsonAsync(); Assert.NotNull(payload); Assert.Equal("NO_SBOM", payload!.ErrorCode); Assert.Equal(2, payload.TotalRuntimeLibraries); Assert.Equal(0, payload.TotalSbomComponents); Assert.Equal(0, payload.MatchCount); Assert.Equal(2, payload.MissCount); Assert.Equal(2, payload.Misses.Count); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconcileEndpoint_WithHashMatches_ReturnsMatches() { var mockObjectStore = new InMemoryArtifactObjectStore(); using var factory = new ScannerApplicationFactory().WithOverrides( configureServices: services => { services.RemoveAll(); services.AddSingleton(mockObjectStore); }); using var client = factory.CreateClient(); // Setup: Create SBOM artifact with components const string sbomArtifactId = "imagebom/sha256-sbomdigest"; const string sbomHash = "sha256:sbomdigest"; using (var scope = factory.Services.CreateScope()) { var artifacts = scope.ServiceProvider.GetRequiredService(); var links = scope.ServiceProvider.GetRequiredService(); await artifacts.UpsertAsync(new ArtifactDocument { Id = sbomArtifactId, Type = ArtifactDocumentType.ImageBom, Format = ArtifactDocumentFormat.CycloneDxJson, MediaType = "application/json", BytesSha256 = sbomHash, RefCount = 1 }, CancellationToken.None); await links.UpsertAsync(new LinkDocument { Id = Guid.NewGuid().ToString("N"), FromType = LinkSourceType.Image, FromDigest = TestImageDigest, ArtifactId = sbomArtifactId, CreatedAtUtc = DateTime.UtcNow }, CancellationToken.None); } // Create SBOM content with matching hash var sbom = CreateSbomWithComponents(new[] { ("comp-1", "openssl", "3.0.0", "pkg:deb/debian/openssl@3.0.0", new[] { "lib1hash" }, new[] { "/lib/libssl.so.3" }), ("comp-2", "libcrypto", "3.0.0", "pkg:deb/debian/libcrypto@3.0.0", new[] { "lib2hash" }, new[] { "/lib/libcrypto.so.3" }) }); var sbomJson = await SerializeSbomAsync(sbom); var sbomBytes = Encoding.UTF8.GetBytes(sbomJson); mockObjectStore.Store($"scanner-artifacts/imagebom/cyclonedx-json/{sbomHash}", sbomBytes); // Ingest runtime event with matching libraries var ingestRequest = new RuntimeEventsIngestRequestDto { Events = new[] { CreateEnvelopeWithLibraries("evt-hash-001", TestImageDigest, new[] { new RuntimeLoadedLibrary { Path = "/lib/libssl.so.3", Sha256 = "lib1hash", Inode = 1001 }, new RuntimeLoadedLibrary { Path = "/lib/libcrypto.so.3", Sha256 = "lib2hash", Inode = 1002 } }) } }; var ingestResponse = await client.PostAsJsonAsync("/api/v1/runtime/events", ingestRequest); Assert.Equal(HttpStatusCode.Accepted, ingestResponse.StatusCode); // Request reconciliation var reconcileRequest = new RuntimeReconcileRequestDto { ImageDigest = TestImageDigest }; var response = await client.PostAsJsonAsync("/api/v1/runtime/reconcile", reconcileRequest); Assert.Equal(HttpStatusCode.OK, response.StatusCode); var payload = await response.Content.ReadFromJsonAsync(); Assert.NotNull(payload); Assert.Null(payload!.ErrorCode); Assert.Equal(2, payload.TotalRuntimeLibraries); Assert.Equal(2, payload.TotalSbomComponents); Assert.Equal(2, payload.MatchCount); Assert.Equal(0, payload.MissCount); Assert.Equal(2, payload.Matches.Count); Assert.All(payload.Matches, m => Assert.Equal("sha256", m.MatchType)); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconcileEndpoint_WithPathMatches_ReturnsMatches() { var mockObjectStore = new InMemoryArtifactObjectStore(); using var factory = new ScannerApplicationFactory().WithOverrides( configureServices: services => { services.RemoveAll(); services.AddSingleton(mockObjectStore); }); using var client = factory.CreateClient(); const string imageDigest = "sha256:pathtest123"; const string sbomArtifactId = "imagebom/sha256-sbomdigest-path"; const string sbomHash = "sha256:sbomdigest-path"; using (var scope = factory.Services.CreateScope()) { var artifacts = scope.ServiceProvider.GetRequiredService(); var links = scope.ServiceProvider.GetRequiredService(); await artifacts.UpsertAsync(new ArtifactDocument { Id = sbomArtifactId, Type = ArtifactDocumentType.ImageBom, Format = ArtifactDocumentFormat.CycloneDxJson, MediaType = "application/json", BytesSha256 = sbomHash, RefCount = 1 }, CancellationToken.None); await links.UpsertAsync(new LinkDocument { Id = Guid.NewGuid().ToString("N"), FromType = LinkSourceType.Image, FromDigest = imageDigest, ArtifactId = sbomArtifactId, CreatedAtUtc = DateTime.UtcNow }, CancellationToken.None); } // Create SBOM with paths but different hashes (path matching) var sbom = CreateSbomWithComponents(new[] { ("comp-1", "zlib", "1.2.11", "pkg:deb/debian/zlib@1.2.11", Array.Empty(), new[] { "/usr/lib/libz.so.1" }) }); var sbomJson = await SerializeSbomAsync(sbom); var sbomBytes = Encoding.UTF8.GetBytes(sbomJson); mockObjectStore.Store($"scanner-artifacts/imagebom/cyclonedx-json/{sbomHash}", sbomBytes); // Ingest runtime event - no hash, path match only var ingestRequest = new RuntimeEventsIngestRequestDto { Events = new[] { CreateEnvelopeWithLibraries("evt-path-001", imageDigest, new[] { new RuntimeLoadedLibrary { Path = "/usr/lib/libz.so.1", Sha256 = null, Inode = 2001 } }) } }; var ingestResponse = await client.PostAsJsonAsync("/api/v1/runtime/events", ingestRequest); Assert.Equal(HttpStatusCode.Accepted, ingestResponse.StatusCode); var reconcileRequest = new RuntimeReconcileRequestDto { ImageDigest = imageDigest }; var response = await client.PostAsJsonAsync("/api/v1/runtime/reconcile", reconcileRequest); Assert.Equal(HttpStatusCode.OK, response.StatusCode); var payload = await response.Content.ReadFromJsonAsync(); Assert.NotNull(payload); Assert.Null(payload!.ErrorCode); Assert.Equal(1, payload.MatchCount); Assert.Equal(0, payload.MissCount); Assert.Single(payload.Matches); Assert.Equal("path", payload.Matches[0].MatchType); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconcileEndpoint_WithSpecificEventId_UsesSpecifiedEvent() { var mockObjectStore = new InMemoryArtifactObjectStore(); using var factory = new ScannerApplicationFactory().WithOverrides( configureServices: services => { services.RemoveAll(); services.AddSingleton(mockObjectStore); }); using var client = factory.CreateClient(); const string imageDigest = "sha256:eventidtest"; const string sbomArtifactId = "imagebom/sha256-sbomdigest-eventid"; const string sbomHash = "sha256:sbomdigest-eventid"; using (var scope = factory.Services.CreateScope()) { var artifacts = scope.ServiceProvider.GetRequiredService(); var links = scope.ServiceProvider.GetRequiredService(); await artifacts.UpsertAsync(new ArtifactDocument { Id = sbomArtifactId, Type = ArtifactDocumentType.ImageBom, Format = ArtifactDocumentFormat.CycloneDxJson, MediaType = "application/json", BytesSha256 = sbomHash, RefCount = 1 }, CancellationToken.None); await links.UpsertAsync(new LinkDocument { Id = Guid.NewGuid().ToString("N"), FromType = LinkSourceType.Image, FromDigest = imageDigest, ArtifactId = sbomArtifactId, CreatedAtUtc = DateTime.UtcNow }, CancellationToken.None); } var sbom = CreateSbomWithComponents(new[] { ("comp-1", "test-lib", "1.0.0", "pkg:test/lib@1.0.0", new[] { "specifichash" }, Array.Empty()) }); var sbomJson = await SerializeSbomAsync(sbom); var sbomBytes = Encoding.UTF8.GetBytes(sbomJson); mockObjectStore.Store($"scanner-artifacts/imagebom/cyclonedx-json/{sbomHash}", sbomBytes); // Ingest multiple events with different libraries var ingestRequest = new RuntimeEventsIngestRequestDto { Events = new[] { CreateEnvelopeWithLibraries("evt-specific-001", imageDigest, new[] { new RuntimeLoadedLibrary { Path = "/lib/specific.so", Sha256 = "specifichash", Inode = 3001 } }), CreateEnvelopeWithLibraries("evt-specific-002", imageDigest, new[] { new RuntimeLoadedLibrary { Path = "/lib/other.so", Sha256 = "otherhash", Inode = 3002 } }) } }; var ingestResponse = await client.PostAsJsonAsync("/api/v1/runtime/events", ingestRequest); Assert.Equal(HttpStatusCode.Accepted, ingestResponse.StatusCode); // Request reconciliation for specific event (evt-specific-001 should match) var reconcileRequest = new RuntimeReconcileRequestDto { ImageDigest = imageDigest, RuntimeEventId = "evt-specific-001" }; var response = await client.PostAsJsonAsync("/api/v1/runtime/reconcile", reconcileRequest); Assert.Equal(HttpStatusCode.OK, response.StatusCode); var payload = await response.Content.ReadFromJsonAsync(); Assert.NotNull(payload); Assert.Equal("evt-specific-001", payload!.RuntimeEventId); Assert.Equal(1, payload.MatchCount); Assert.Equal(0, payload.MissCount); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconcileEndpoint_WithNonExistentEventId_ReturnsNotFound() { using var factory = new ScannerApplicationFactory(); using var client = factory.CreateClient(); var request = new RuntimeReconcileRequestDto { ImageDigest = TestImageDigest, RuntimeEventId = "non-existent-event-id" }; var response = await client.PostAsJsonAsync("/api/v1/runtime/reconcile", request); Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); var payload = await response.Content.ReadFromJsonAsync(); Assert.NotNull(payload); Assert.Equal("RUNTIME_EVENT_NOT_FOUND", payload!.ErrorCode); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconcileEndpoint_WithMissingImageDigest_ReturnsBadRequest() { using var factory = new ScannerApplicationFactory(); using var client = factory.CreateClient(); var request = new RuntimeReconcileRequestDto { ImageDigest = "" }; var response = await client.PostAsJsonAsync("/api/v1/runtime/reconcile", request); Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task ReconcileEndpoint_WithMixedMatchesAndMisses_ReturnsCorrectCounts() { var mockObjectStore = new InMemoryArtifactObjectStore(); using var factory = new ScannerApplicationFactory().WithOverrides( configureServices: services => { services.RemoveAll(); services.AddSingleton(mockObjectStore); }); using var client = factory.CreateClient(); const string imageDigest = "sha256:mixedtest"; const string sbomArtifactId = "imagebom/sha256-sbomdigest-mixed"; const string sbomHash = "sha256:sbomdigest-mixed"; using (var scope = factory.Services.CreateScope()) { var artifacts = scope.ServiceProvider.GetRequiredService(); var links = scope.ServiceProvider.GetRequiredService(); await artifacts.UpsertAsync(new ArtifactDocument { Id = sbomArtifactId, Type = ArtifactDocumentType.ImageBom, Format = ArtifactDocumentFormat.CycloneDxJson, MediaType = "application/json", BytesSha256 = sbomHash, RefCount = 1 }, CancellationToken.None); await links.UpsertAsync(new LinkDocument { Id = Guid.NewGuid().ToString("N"), FromType = LinkSourceType.Image, FromDigest = imageDigest, ArtifactId = sbomArtifactId, CreatedAtUtc = DateTime.UtcNow }, CancellationToken.None); } // SBOM has 2 components var sbom = CreateSbomWithComponents(new[] { ("comp-known-1", "known-lib", "1.0.0", "pkg:test/known@1.0.0", new[] { "knownhash1" }, new[] { "/lib/known.so" }), ("comp-known-2", "another-lib", "2.0.0", "pkg:test/another@2.0.0", new[] { "knownhash2" }, Array.Empty()) }); var sbomJson = await SerializeSbomAsync(sbom); var sbomBytes = Encoding.UTF8.GetBytes(sbomJson); mockObjectStore.Store($"scanner-artifacts/imagebom/cyclonedx-json/{sbomHash}", sbomBytes); // Runtime has 3 libraries: 1 hash match, 1 path match, 1 miss var ingestRequest = new RuntimeEventsIngestRequestDto { Events = new[] { CreateEnvelopeWithLibraries("evt-mixed-001", imageDigest, new[] { new RuntimeLoadedLibrary { Path = "/lib/known.so", Sha256 = "knownhash1", Inode = 4001 }, // hash match new RuntimeLoadedLibrary { Path = "/lib/unknown.so", Sha256 = "unknownhash", Inode = 4002 }, // miss new RuntimeLoadedLibrary { Path = "/lib/another.so", Sha256 = "knownhash2", Inode = 4003 } // hash match }) } }; var ingestResponse = await client.PostAsJsonAsync("/api/v1/runtime/events", ingestRequest); Assert.Equal(HttpStatusCode.Accepted, ingestResponse.StatusCode); var reconcileRequest = new RuntimeReconcileRequestDto { ImageDigest = imageDigest }; var response = await client.PostAsJsonAsync("/api/v1/runtime/reconcile", reconcileRequest); Assert.Equal(HttpStatusCode.OK, response.StatusCode); var payload = await response.Content.ReadFromJsonAsync(); Assert.NotNull(payload); Assert.Null(payload!.ErrorCode); Assert.Equal(3, payload.TotalRuntimeLibraries); Assert.Equal(2, payload.TotalSbomComponents); Assert.Equal(2, payload.MatchCount); Assert.Equal(1, payload.MissCount); Assert.Single(payload.Misses); Assert.Equal("/lib/unknown.so", payload.Misses[0].Path); } private static RuntimeEventEnvelope CreateEnvelopeWithLibraries( string eventId, string imageDigest, RuntimeLoadedLibrary[] libraries) { var runtimeEvent = new RuntimeEvent { EventId = eventId, When = DateTimeOffset.UtcNow, Kind = RuntimeEventKind.ContainerStart, Tenant = TestTenant, Node = TestNode, Runtime = new RuntimeEngine { Engine = "containerd", Version = "1.7.0" }, Workload = new RuntimeWorkload { Platform = "kubernetes", Namespace = "default", Pod = "test-pod", Container = "test-container", ContainerId = $"containerd://{eventId}", ImageRef = $"ghcr.io/example/test@{imageDigest}" }, Delta = new RuntimeDelta { BaselineImageDigest = imageDigest }, Process = new RuntimeProcess { Pid = 1234, Entrypoint = new[] { "/bin/start" }, EntryTrace = Array.Empty() }, LoadedLibraries = libraries }; return RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); } private static Bom CreateSbomWithComponents( (string bomRef, string name, string version, string purl, string[] hashes, string[] paths)[] components) { var bom = new Bom { Version = 1, SerialNumber = $"urn:uuid:{Guid.NewGuid()}", Components = new List() }; foreach (var (bomRef, name, version, purl, hashes, paths) in components) { var component = new Component { BomRef = bomRef, Name = name, Version = version, Purl = purl, Type = Component.Classification.Library, Hashes = hashes.Select(h => new Hash { Alg = Hash.HashAlgorithm.SHA_256, Content = h }).ToList() }; if (paths.Length > 0) { component.Evidence = new CycloneDX.Models.Evidence { Occurrences = paths.Select(p => new EvidenceOccurrence { Location = p }).ToList() }; } bom.Components.Add(component); } return bom; } private static async Task SerializeSbomAsync(Bom sbom) { await using var buffer = new MemoryStream(); await Serializer.SerializeAsync(sbom, buffer); return Encoding.UTF8.GetString(buffer.ToArray()); } private sealed class InMemoryArtifactObjectStore : IArtifactObjectStore { private readonly Dictionary _store = new(StringComparer.OrdinalIgnoreCase); public void Store(string key, byte[] content) { _store[key] = content; } public Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken) { using var ms = new MemoryStream(); content.CopyTo(ms); _store[$"{descriptor.Bucket}/{descriptor.Key}"] = ms.ToArray(); return Task.CompletedTask; } public Task GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken) { var key = $"{descriptor.Bucket}/{descriptor.Key}"; if (_store.TryGetValue(key, out var content)) { return Task.FromResult(new MemoryStream(content)); } return Task.FromResult(null); } public Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken) { var key = $"{descriptor.Bucket}/{descriptor.Key}"; _store.Remove(key); return Task.CompletedTask; } } }