diff --git a/docs/implplan/SPRINT_3105_0001_0001_proofspine_cbor_accept.md b/docs/implplan/SPRINT_3105_0001_0001_proofspine_cbor_accept.md index 28da1a610..3ce3999b6 100644 --- a/docs/implplan/SPRINT_3105_0001_0001_proofspine_cbor_accept.md +++ b/docs/implplan/SPRINT_3105_0001_0001_proofspine_cbor_accept.md @@ -21,9 +21,9 @@ | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | | 1 | PROOF-CBOR-3105-001 | DOING | ProofSpine endpoints | Scanner · WebService | Add `Accept: application/cbor` support to ProofSpine endpoints with deterministic encoding. | -| 2 | PROOF-CBOR-3105-002 | TODO | Encoder helper | Scanner · WebService | Add a shared CBOR encoder helper (JSON→CBOR) with stable key ordering. | -| 3 | PROOF-CBOR-3105-003 | TODO | Integration tests | Scanner · QA | Add endpoint tests validating CBOR content-type and decoding key fields. | -| 4 | PROOF-CBOR-3105-004 | TODO | Close bookkeeping | Scanner · WebService | Update local `TASKS.md`, sprint status, and execution log with evidence (test run). | +| 2 | PROOF-CBOR-3105-002 | DOING | Encoder helper | Scanner · WebService | Add a shared CBOR encoder helper (JSON→CBOR) with stable key ordering. | +| 3 | PROOF-CBOR-3105-003 | DOING | Integration tests | Scanner · QA | Add endpoint tests validating CBOR content-type and decoding key fields. | +| 4 | PROOF-CBOR-3105-004 | DOING | Close bookkeeping | Scanner · WebService | Update local `TASKS.md`, sprint status, and execution log with evidence (test run). | ## Decisions & Risks - **Decision:** CBOR payload shape matches JSON DTO shape (same property names). @@ -33,4 +33,4 @@ | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-12-18 | Sprint created; started PROOF-CBOR-3105-001. | Agent | - +| 2025-12-18 | Started PROOF-CBOR-3105-002..004. | Agent | diff --git a/src/Scanner/StellaOps.Scanner.WebService/Endpoints/ProofSpineEndpoints.cs b/src/Scanner/StellaOps.Scanner.WebService/Endpoints/ProofSpineEndpoints.cs index 51bf75b3b..933a9fecb 100644 --- a/src/Scanner/StellaOps.Scanner.WebService/Endpoints/ProofSpineEndpoints.cs +++ b/src/Scanner/StellaOps.Scanner.WebService/Endpoints/ProofSpineEndpoints.cs @@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Routing; using StellaOps.Replay.Core; using StellaOps.Scanner.ProofSpine; using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Serialization; using StellaOps.Scanner.WebService.Security; namespace StellaOps.Scanner.WebService.Endpoints; @@ -17,6 +18,7 @@ internal static class ProofSpineEndpoints spines.MapGet("/{spineId}", HandleGetSpineAsync) .WithName("scanner.spines.get") .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status200OK, contentType: CborNegotiation.ContentType) .Produces(StatusCodes.Status404NotFound) .RequireAuthorization(ScannerPolicies.ScansRead); @@ -24,15 +26,18 @@ internal static class ProofSpineEndpoints scans.MapGet("/{scanId}/spines", HandleListSpinesAsync) .WithName("scanner.spines.list-by-scan") .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status200OK, contentType: CborNegotiation.ContentType) .RequireAuthorization(ScannerPolicies.ScansRead); } private static async Task HandleGetSpineAsync( + HttpRequest request, string spineId, IProofSpineRepository repository, ProofSpineVerifier verifier, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(request); ArgumentNullException.ThrowIfNull(repository); ArgumentNullException.ThrowIfNull(verifier); @@ -93,19 +98,36 @@ internal static class ProofSpineEndpoints } }; + if (CborNegotiation.AcceptsCbor(request)) + { + return Results.Bytes( + DeterministicCborSerializer.Serialize(dto), + contentType: CborNegotiation.ContentType); + } + return Results.Ok(dto); } private static async Task HandleListSpinesAsync( + HttpRequest request, string scanId, IProofSpineRepository repository, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(request); ArgumentNullException.ThrowIfNull(repository); if (string.IsNullOrWhiteSpace(scanId)) { - return Results.Ok(new ProofSpineListResponseDto { Items = Array.Empty(), Total = 0 }); + var empty = new ProofSpineListResponseDto { Items = Array.Empty(), Total = 0 }; + if (CborNegotiation.AcceptsCbor(request)) + { + return Results.Bytes( + DeterministicCborSerializer.Serialize(empty), + contentType: CborNegotiation.ContentType); + } + + return Results.Ok(empty); } var summaries = await repository.GetSummariesByScanRunAsync(scanId, cancellationToken).ConfigureAwait(false); @@ -119,11 +141,20 @@ internal static class ProofSpineEndpoints CreatedAt = summary.CreatedAt }).ToArray(); - return Results.Ok(new ProofSpineListResponseDto + var response = new ProofSpineListResponseDto { Items = items, Total = items.Length - }); + }; + + if (CborNegotiation.AcceptsCbor(request)) + { + return Results.Bytes( + DeterministicCborSerializer.Serialize(response), + contentType: CborNegotiation.ContentType); + } + + return Results.Ok(response); } private static DsseEnvelopeDto MapEnvelope(DsseEnvelope envelope) @@ -163,4 +194,3 @@ internal static class ProofSpineEndpoints return "/" + trimmed; } } - diff --git a/src/Scanner/StellaOps.Scanner.WebService/Serialization/CborNegotiation.cs b/src/Scanner/StellaOps.Scanner.WebService/Serialization/CborNegotiation.cs new file mode 100644 index 000000000..58550d48a --- /dev/null +++ b/src/Scanner/StellaOps.Scanner.WebService/Serialization/CborNegotiation.cs @@ -0,0 +1,29 @@ +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Scanner.WebService.Serialization; + +internal static class CborNegotiation +{ + public const string ContentType = "application/cbor"; + + public static bool AcceptsCbor(HttpRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + if (!request.Headers.TryGetValue("Accept", out var values)) + { + return false; + } + + foreach (var value in values) + { + if (value.Contains(ContentType, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } +} + diff --git a/src/Scanner/StellaOps.Scanner.WebService/Serialization/DeterministicCborSerializer.cs b/src/Scanner/StellaOps.Scanner.WebService/Serialization/DeterministicCborSerializer.cs new file mode 100644 index 000000000..817a47ccb --- /dev/null +++ b/src/Scanner/StellaOps.Scanner.WebService/Serialization/DeterministicCborSerializer.cs @@ -0,0 +1,108 @@ +using System.Collections.Generic; +using System.Formats.Cbor; +using System.Text; +using System.Text.Json; + +namespace StellaOps.Scanner.WebService.Serialization; + +internal static class DeterministicCborSerializer +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = false + }; + + public static byte[] Serialize(T value) + { + using var document = JsonSerializer.SerializeToDocument(value, JsonOptions); + var writer = new CborWriter(CborConformanceMode.Canonical); + + WriteElement(writer, document.RootElement); + return writer.Encode(); + } + + private static void WriteElement(CborWriter writer, JsonElement element) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + WriteObject(writer, element); + return; + case JsonValueKind.Array: + writer.WriteStartArray(element.GetArrayLength()); + foreach (var item in element.EnumerateArray()) + { + WriteElement(writer, item); + } + writer.WriteEndArray(); + return; + case JsonValueKind.String: + writer.WriteTextString(element.GetString() ?? string.Empty); + return; + case JsonValueKind.Number: + if (element.TryGetInt64(out var int64)) + { + writer.WriteInt64(int64); + return; + } + + writer.WriteDouble(element.GetDouble()); + return; + case JsonValueKind.True: + writer.WriteBoolean(true); + return; + case JsonValueKind.False: + writer.WriteBoolean(false); + return; + case JsonValueKind.Null: + case JsonValueKind.Undefined: + writer.WriteNull(); + return; + default: + writer.WriteNull(); + return; + } + } + + private static void WriteObject(CborWriter writer, JsonElement element) + { + var properties = new List<(byte[] KeyBytes, string Key, JsonElement Value)>(); + + foreach (var property in element.EnumerateObject()) + { + var keyBytes = Encoding.UTF8.GetBytes(property.Name); + properties.Add((keyBytes, property.Name, property.Value)); + } + + properties.Sort(static (left, right) => + { + var lengthCompare = left.KeyBytes.Length.CompareTo(right.KeyBytes.Length); + if (lengthCompare != 0) + { + return lengthCompare; + } + + for (var i = 0; i < left.KeyBytes.Length; i++) + { + var byteCompare = left.KeyBytes[i].CompareTo(right.KeyBytes[i]); + if (byteCompare != 0) + { + return byteCompare; + } + } + + return 0; + }); + + writer.WriteStartMap(properties.Count); + + foreach (var (_, key, value) in properties) + { + writer.WriteTextString(key); + WriteElement(writer, value); + } + + writer.WriteEndMap(); + } +} + diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/EpssSignalJob.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/EpssSignalJob.cs index f4c9957a4..07d42614f 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Processing/EpssSignalJob.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/EpssSignalJob.cs @@ -404,7 +404,7 @@ public sealed class EpssSignalJob : BackgroundService return EpssSignalEventTypes.RiskSpike; } - if (change.Flags.HasFlag(EpssChangeFlags.DroppedLow)) + if (change.Flags.HasFlag(EpssChangeFlags.BigJumpDown)) { return EpssSignalEventTypes.DroppedLow; } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Epss/EpssPriorityBand.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Epss/EpssPriorityBand.cs index cd0af1c72..3eb73a1d3 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Epss/EpssPriorityBand.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Epss/EpssPriorityBand.cs @@ -124,7 +124,7 @@ public sealed class EpssPriorityCalculator var flags = EpssChangeFlags.None; // NEW_SCORED: first time we have EPSS data - if (!oldScore.HasValue && newScore > 0) + if (!oldScore.HasValue) { flags |= EpssChangeFlags.NewScored; } @@ -133,31 +133,37 @@ public sealed class EpssPriorityCalculator { var delta = newScore - oldScore.Value; - // BIG_JUMP: significant score increase - if (delta >= _options.BigJumpDelta) - { - flags |= EpssChangeFlags.BigJump; - } - - // DROPPED_LOW: significant score decrease - if (delta <= -_options.DroppedLowDelta) - { - flags |= EpssChangeFlags.DroppedLow; - } - } - - // CROSSED_HIGH: moved into or out of high priority - var oldBand = Calculate(oldScore, oldPercentile).Band; - var newBand = Calculate(newScore, newPercentile).Band; - - if (oldBand != newBand) - { - // Crossed into critical or high - if ((newBand == EpssPriorityBand.Critical || newBand == EpssPriorityBand.High) && - oldBand != EpssPriorityBand.Critical && oldBand != EpssPriorityBand.High) + if (oldScore.Value < _options.HighScore && newScore >= _options.HighScore) { flags |= EpssChangeFlags.CrossedHigh; } + + if (oldScore.Value >= _options.HighScore && newScore < _options.HighScore) + { + flags |= EpssChangeFlags.CrossedLow; + } + + if (delta > _options.BigJumpDelta) + { + flags |= EpssChangeFlags.BigJumpUp; + } + + if (delta < -_options.DroppedLowDelta) + { + flags |= EpssChangeFlags.BigJumpDown; + } + } + + if ((oldPercentile is null || oldPercentile < _options.HighPercentile) + && newPercentile >= _options.HighPercentile) + { + flags |= EpssChangeFlags.TopPercentile; + } + + if (oldPercentile is not null && oldPercentile >= _options.HighPercentile + && newPercentile < _options.HighPercentile) + { + flags |= EpssChangeFlags.LeftTopPercentile; } return flags; @@ -174,14 +180,23 @@ public enum EpssChangeFlags None = 0, /// CVE was scored for the first time. - NewScored = 1 << 0, + NewScored = 1, /// Score crossed into high priority band. - CrossedHigh = 1 << 1, + CrossedHigh = 2, + + /// Score crossed below the high score threshold. + CrossedLow = 4, /// Score increased significantly (above BigJumpDelta). - BigJump = 1 << 2, + BigJumpUp = 8, - /// Score dropped significantly (above DroppedLowDelta). - DroppedLow = 1 << 3 + /// Score decreased significantly (above DroppedLowDelta). + BigJumpDown = 16, + + /// Entered the top percentile band. + TopPercentile = 32, + + /// Left the top percentile band. + LeftTopPercentile = 64 } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/CachingEpssProvider.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/CachingEpssProvider.cs index 26c379134..6376261cb 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/CachingEpssProvider.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/CachingEpssProvider.cs @@ -8,6 +8,7 @@ using System.Diagnostics; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using StellaOps.Messaging; using StellaOps.Messaging.Abstractions; using StellaOps.Scanner.Core.Epss; @@ -58,7 +59,7 @@ public sealed class CachingEpssProvider : IEpssProvider { var cacheResult = await _cache.GetAsync(cacheKey, cancellationToken).ConfigureAwait(false); - if (cacheResult.IsHit && cacheResult.Value is not null) + if (cacheResult.HasValue && cacheResult.Value is not null) { _logger.LogDebug("Cache hit for EPSS score: {CveId}", cveId); return MapFromCacheEntry(cacheResult.Value, fromCache: true); @@ -119,7 +120,7 @@ public sealed class CachingEpssProvider : IEpssProvider var cacheKey = BuildCacheKey(cveId); var cacheResult = await _cache.GetAsync(cacheKey, cancellationToken).ConfigureAwait(false); - if (cacheResult.IsHit && cacheResult.Value is not null) + if (cacheResult.HasValue && cacheResult.Value is not null) { var evidence = MapFromCacheEntry(cacheResult.Value, fromCache: true); found.Add(evidence); @@ -214,7 +215,7 @@ public sealed class CachingEpssProvider : IEpssProvider { var cacheResult = await _cache.GetAsync(ModelDateCacheKey, cancellationToken).ConfigureAwait(false); - if (cacheResult.IsHit && cacheResult.Value?.ModelDate is not null) + if (cacheResult.HasValue && cacheResult.Value is not null) { return cacheResult.Value.ModelDate; } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/EpssChangeDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/EpssChangeDetector.cs index 6d9dfd02b..e84253867 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/EpssChangeDetector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/EpssChangeDetector.cs @@ -5,6 +5,8 @@ // Description: Deterministic EPSS delta flag computation (mirrors SQL function). // ----------------------------------------------------------------------------- +using StellaOps.Scanner.Core.Epss; + namespace StellaOps.Scanner.Storage.Epss; public static class EpssChangeDetector @@ -72,4 +74,3 @@ public readonly record struct EpssChangeThresholds( double HighScore, double HighPercentile, double BigJumpDelta); - diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/EpssChangeFlags.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/EpssChangeFlags.cs deleted file mode 100644 index a27dad044..000000000 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Epss/EpssChangeFlags.cs +++ /dev/null @@ -1,36 +0,0 @@ -// ----------------------------------------------------------------------------- -// EpssChangeFlags.cs -// Sprint: SPRINT_3410_0001_0001_epss_ingestion_storage -// Task: EPSS-3410-008 -// Description: Flag bitmask for EPSS change detection. -// ----------------------------------------------------------------------------- - -namespace StellaOps.Scanner.Storage.Epss; - -[Flags] -public enum EpssChangeFlags -{ - None = 0, - - /// 0x01 - CVE newly scored (not in previous snapshot). - NewScored = 1, - - /// 0x02 - Crossed above the high score threshold. - CrossedHigh = 2, - - /// 0x04 - Crossed below the high score threshold. - CrossedLow = 4, - - /// 0x08 - Score increased by more than the big jump delta. - BigJumpUp = 8, - - /// 0x10 - Score decreased by more than the big jump delta. - BigJumpDown = 16, - - /// 0x20 - Entered the top percentile band. - TopPercentile = 32, - - /// 0x40 - Left the top percentile band. - LeftTopPercentile = 64 -} - diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Extensions/EpssServiceCollectionExtensions.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Extensions/EpssServiceCollectionExtensions.cs index 9c4e5a352..96ac103c7 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Extensions/EpssServiceCollectionExtensions.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Extensions/EpssServiceCollectionExtensions.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using StellaOps.Messaging; using StellaOps.Messaging.Abstractions; using StellaOps.Scanner.Core.Epss; using StellaOps.Scanner.Storage.Epss; diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRawRepository.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRawRepository.cs index 4913ebeee..f86b0b1c0 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRawRepository.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRawRepository.cs @@ -44,7 +44,7 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository RETURNING raw_id, ingestion_ts """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); var result = await connection.QueryFirstOrDefaultAsync<(long raw_id, DateTimeOffset ingestion_ts)?>(sql, new { @@ -88,7 +88,7 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository LIMIT 1 """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); var row = await connection.QueryFirstOrDefaultAsync(sql, new { AsOfDate = asOfDate.ToDateTime(TimeOnly.MinValue) @@ -112,7 +112,7 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository ORDER BY asof_date DESC """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); var rows = await connection.QueryAsync(sql, new { StartDate = startDate.ToDateTime(TimeOnly.MinValue), @@ -134,7 +134,7 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository LIMIT 1 """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); var row = await connection.QueryFirstOrDefaultAsync(sql); return row.HasValue ? MapToRaw(row.Value) : null; @@ -149,7 +149,7 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository ) """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); return await connection.ExecuteScalarAsync(sql, new { AsOfDate = asOfDate.ToDateTime(TimeOnly.MinValue), @@ -173,7 +173,7 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository LIMIT @Limit """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); var rows = await connection.QueryAsync(sql, new { ModelVersion = modelVersion, @@ -187,7 +187,7 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository { var sql = $"SELECT {SchemaName}.prune_epss_raw(@RetentionDays)"; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); return await connection.ExecuteScalarAsync(sql, new { RetentionDays = retentionDays }); } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRepository.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRepository.cs index 2312ced28..5ab431f90 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRepository.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRepository.cs @@ -505,14 +505,17 @@ public sealed class PostgresEpssRepository : IEpssRepository LIMIT @Limit """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false); - var rows = await connection.QueryAsync(sql, new - { - ModelDate = modelDate, - Flags = flags.HasValue ? (int)flags.Value : 0, - Limit = limit - }); + var rows = await connection.QueryAsync(new CommandDefinition( + sql, + new + { + ModelDate = modelDate, + Flags = flags.HasValue ? (int)flags.Value : 0, + Limit = limit + }, + cancellationToken: cancellationToken)).ConfigureAwait(false); return rows.Select(r => new EpssChangeRecord { diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssSignalRepository.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssSignalRepository.cs index 7b4f22187..7d2331aae 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssSignalRepository.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssSignalRepository.cs @@ -46,7 +46,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository RETURNING signal_id, created_at """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(signal.TenantId.ToString("D"), cancellationToken); var result = await connection.QueryFirstOrDefaultAsync<(long signal_id, DateTimeOffset created_at)?>(sql, new { @@ -104,34 +104,38 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository ON CONFLICT (tenant_id, dedupe_key) DO NOTHING """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); - await using var transaction = await connection.BeginTransactionAsync(cancellationToken); - var inserted = 0; - foreach (var signal in signalList) + foreach (var tenantGroup in signalList.GroupBy(signal => signal.TenantId)) { - var affected = await connection.ExecuteAsync(sql, new - { - signal.TenantId, - ModelDate = signal.ModelDate.ToDateTime(TimeOnly.MinValue), - signal.CveId, - signal.EventType, - signal.RiskBand, - signal.EpssScore, - signal.EpssDelta, - signal.Percentile, - signal.PercentileDelta, - signal.IsModelChange, - signal.ModelVersion, - signal.DedupeKey, - signal.ExplainHash, - signal.Payload - }, transaction); + await using var connection = await _dataSource.OpenConnectionAsync(tenantGroup.Key.ToString("D"), cancellationToken); + await using var transaction = await connection.BeginTransactionAsync(cancellationToken); - inserted += affected; + foreach (var signal in tenantGroup) + { + var affected = await connection.ExecuteAsync(sql, new + { + signal.TenantId, + ModelDate = signal.ModelDate.ToDateTime(TimeOnly.MinValue), + signal.CveId, + signal.EventType, + signal.RiskBand, + signal.EpssScore, + signal.EpssDelta, + signal.Percentile, + signal.PercentileDelta, + signal.IsModelChange, + signal.ModelVersion, + signal.DedupeKey, + signal.ExplainHash, + signal.Payload + }, transaction); + + inserted += affected; + } + + await transaction.CommitAsync(cancellationToken); } - await transaction.CommitAsync(cancellationToken); return inserted; } @@ -159,7 +163,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository LIMIT 10000 """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken); var rows = await connection.QueryAsync(sql, new { @@ -190,7 +194,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository LIMIT @Limit """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken); var rows = await connection.QueryAsync(sql, new { @@ -222,7 +226,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository LIMIT 10000 """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken); var rows = await connection.QueryAsync(sql, new { @@ -246,7 +250,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository WHERE tenant_id = @TenantId """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken); var row = await connection.QueryFirstOrDefaultAsync(sql, new { TenantId = tenantId }); @@ -277,7 +281,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository RETURNING config_id, created_at, updated_at """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(config.TenantId.ToString("D"), cancellationToken); var result = await connection.QueryFirstAsync<(Guid config_id, DateTimeOffset created_at, DateTimeOffset updated_at)>(sql, new { @@ -302,7 +306,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository { var sql = $"SELECT {SchemaName}.prune_epss_signals(@RetentionDays)"; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); return await connection.ExecuteScalarAsync(sql, new { RetentionDays = retentionDays }); } @@ -317,7 +321,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository WHERE tenant_id = @TenantId AND dedupe_key = @DedupeKey """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken); var row = await connection.QueryFirstOrDefaultAsync(sql, new { TenantId = tenantId, DedupeKey = dedupeKey }); return row.HasValue ? MapToSignal(row.Value) : null; diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresObservedCveRepository.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresObservedCveRepository.cs index 0904524c8..fc078ff6a 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresObservedCveRepository.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresObservedCveRepository.cs @@ -38,7 +38,7 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository AND cve_id LIKE 'CVE-%' """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken); var cves = await connection.QueryAsync(sql, new { TenantId = tenantId }); return new HashSet(cves, StringComparer.OrdinalIgnoreCase); @@ -52,12 +52,12 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository var sql = $""" SELECT EXISTS ( SELECT 1 FROM {TriageTable} - WHERE tenant_id = @TenantId - AND cve_id = @CveId + WHERE tenant_id = @TenantId + AND cve_id = @CveId ) """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken); return await connection.ExecuteScalarAsync(sql, new { TenantId = tenantId, CveId = cveId }); } @@ -79,7 +79,7 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository AND cve_id = ANY(@CveIds) """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken); var observed = await connection.QueryAsync(sql, new { TenantId = tenantId, @@ -99,7 +99,7 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository AND cve_id LIKE 'CVE-%' """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); var tenants = await connection.QueryAsync(sql); return tenants.ToList(); @@ -122,7 +122,7 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository GROUP BY cve_id, tenant_id """; - await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken); + await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken); var rows = await connection.QueryAsync<(string cve_id, Guid tenant_id)>(sql, new { CveIds = cveList.ToArray() diff --git a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ProofSpineEndpointsTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ProofSpineEndpointsTests.cs index 1a2e91868..7d9c1f1b9 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ProofSpineEndpointsTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ProofSpineEndpointsTests.cs @@ -1,5 +1,8 @@ using System.Collections.Generic; +using System.Formats.Cbor; using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; using Microsoft.Extensions.DependencyInjection; @@ -10,6 +13,8 @@ namespace StellaOps.Scanner.WebService.Tests; public sealed class ProofSpineEndpointsTests { + private const string CborContentType = "application/cbor"; + [Fact] public async Task GetSpine_ReturnsSpine_WithVerification() { @@ -49,6 +54,42 @@ public sealed class ProofSpineEndpointsTests Assert.True(body.TryGetProperty("verification", out _)); } + [Fact] + public async Task GetSpine_ReturnsCbor_WhenAcceptHeaderRequestsCbor() + { + await using var factory = new ScannerApplicationFactory(); + using var scope = factory.Services.CreateScope(); + + var builder = scope.ServiceProvider.GetRequiredService(); + var repository = scope.ServiceProvider.GetRequiredService(); + + var spine = await builder + .ForArtifact("sha256:feedface") + .ForVulnerability("CVE-2025-1001") + .WithPolicyProfile("default") + .WithScanRun("scan-cbor-001") + .AddSbomSlice("sha256:sbom", new[] { "pkg:a", "pkg:b" }, toolId: "sbom", toolVersion: "1.0.0") + .BuildAsync(); + + await repository.SaveAsync(spine); + + var client = factory.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, $"/api/v1/spines/{spine.SpineId}"); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(CborContentType)); + + using var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(CborContentType, response.Content.Headers.ContentType?.MediaType); + + var cbor = await response.Content.ReadAsByteArrayAsync(); + var decoded = ReadRootMap(cbor); + + Assert.Equal(spine.SpineId, decoded["spineId"]); + Assert.True(decoded.ContainsKey("verification")); + Assert.True(((List)decoded["segments"]!).Count > 0); + } + [Fact] public async Task ListSpinesByScan_ReturnsSummaries_WithSegmentCount() { @@ -87,6 +128,44 @@ public sealed class ProofSpineEndpointsTests Assert.True(items[0].GetProperty("segmentCount").GetInt32() > 0); } + [Fact] + public async Task ListSpinesByScan_ReturnsCbor_WhenAcceptHeaderRequestsCbor() + { + await using var factory = new ScannerApplicationFactory(); + using var scope = factory.Services.CreateScope(); + + var builder = scope.ServiceProvider.GetRequiredService(); + var repository = scope.ServiceProvider.GetRequiredService(); + + var spine = await builder + .ForArtifact("sha256:feedface") + .ForVulnerability("CVE-2025-1002") + .WithPolicyProfile("default") + .WithScanRun("scan-cbor-002") + .AddSbomSlice("sha256:sbom", new[] { "pkg:a" }, toolId: "sbom", toolVersion: "1.0.0") + .BuildAsync(); + + await repository.SaveAsync(spine); + + var client = factory.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/v1/scans/scan-cbor-002/spines"); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(CborContentType)); + + using var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(CborContentType, response.Content.Headers.ContentType?.MediaType); + + var cbor = await response.Content.ReadAsByteArrayAsync(); + var decoded = ReadRootMap(cbor); + var items = (List)decoded["items"]!; + + Assert.Single(items); + var first = (Dictionary)items[0]!; + Assert.Equal(spine.SpineId, first["spineId"]); + Assert.True((int)first["segmentCount"]! > 0); + } + [Fact] public async Task GetSpine_ReturnsInvalidStatus_WhenSegmentTampered() { @@ -125,4 +204,90 @@ public sealed class ProofSpineEndpointsTests var segments = body.GetProperty("segments"); Assert.Equal("invalid", segments[0].GetProperty("status").GetString()); } + + private static Dictionary ReadRootMap(byte[] cbor) + { + var reader = new CborReader(cbor, CborConformanceMode.Canonical); + return ReadMap(reader); + } + + private static Dictionary ReadMap(CborReader reader) + { + var length = reader.ReadStartMap(); + var result = new Dictionary(StringComparer.Ordinal); + + if (length is not null) + { + for (var i = 0; i < length.Value; i++) + { + var key = reader.ReadTextString(); + result[key] = ReadValue(reader); + } + reader.ReadEndMap(); + return result; + } + + while (reader.PeekState() != CborReaderState.EndMap) + { + var key = reader.ReadTextString(); + result[key] = ReadValue(reader); + } + + reader.ReadEndMap(); + return result; + } + + private static List ReadArray(CborReader reader) + { + var length = reader.ReadStartArray(); + var result = new List(); + + if (length is not null) + { + for (var i = 0; i < length.Value; i++) + { + result.Add(ReadValue(reader)); + } + reader.ReadEndArray(); + return result; + } + + while (reader.PeekState() != CborReaderState.EndArray) + { + result.Add(ReadValue(reader)); + } + + reader.ReadEndArray(); + return result; + } + + private static object? ReadValue(CborReader reader) + { + switch (reader.PeekState()) + { + case CborReaderState.StartMap: + return ReadMap(reader); + case CborReaderState.StartArray: + return ReadArray(reader); + case CborReaderState.TextString: + return reader.ReadTextString(); + case CborReaderState.UnsignedInteger: + return (int)reader.ReadUInt64(); + case CborReaderState.NegativeInteger: + return (int)reader.ReadInt64(); + case CborReaderState.DoublePrecisionFloat: + return reader.ReadDouble(); + case CborReaderState.SinglePrecisionFloat: + return reader.ReadSingle(); + case CborReaderState.HalfPrecisionFloat: + return reader.ReadHalf(); + case CborReaderState.Boolean: + return reader.ReadBoolean(); + case CborReaderState.Null: + reader.ReadNull(); + return null; + default: + throw new InvalidOperationException($"Unexpected CBOR state {reader.PeekState()}"); + } + } } diff --git a/src/Scanner/docs/events/samples/scanner.event.report.ready@1.sample.json b/src/Scanner/docs/events/samples/scanner.event.report.ready@1.sample.json index 814435d56..bd1f67406 100644 --- a/src/Scanner/docs/events/samples/scanner.event.report.ready@1.sample.json +++ b/src/Scanner/docs/events/samples/scanner.event.report.ready@1.sample.json @@ -64,7 +64,8 @@ "payload": "eyJyZXBvcnRJZCI6InJlcG9ydC1hYmMiLCJpbWFnZURpZ2VzdCI6InNoYTI1NjpmZWVkZmFjZSIsImdlbmVyYXRlZEF0IjoiMjAyNS0xMC0xOVQxMjozNDo1NiswMDowMCIsInZlcmRpY3QiOiJibG9ja2VkIiwicG9saWN5Ijp7InJldmlzaW9uSWQiOiJyZXYtNDIiLCJkaWdlc3QiOiJkaWdlc3QtMTIzIn0sInN1bW1hcnkiOnsidG90YWwiOjEsImJsb2NrZWQiOjEsIndhcm5lZCI6MCwiaWdub3JlZCI6MCwicXVpZXRlZCI6MH0sInZlcmRpY3RzIjpbeyJmaW5kaW5nSWQiOiJmaW5kaW5nLTEiLCJzdGF0dXMiOiJCbG9ja2VkIiwic2NvcmUiOjQ3LjUsInNvdXJjZVRydXN0IjoiTlZEIiwicmVhY2hhYmlsaXR5IjoicnVudGltZSJ9XSwiaXNzdWVzIjpbXX0=", "signatures": [ { - "keyId": "test-key", + "keyid": "test-key", + "sig": "", "algorithm": "hs256", "signature": "signature-value" } diff --git a/src/Scanner/docs/events/samples/scanner.event.scan.completed@1.sample.json b/src/Scanner/docs/events/samples/scanner.event.scan.completed@1.sample.json index c2f416b9b..28b8e9fc8 100644 --- a/src/Scanner/docs/events/samples/scanner.event.scan.completed@1.sample.json +++ b/src/Scanner/docs/events/samples/scanner.event.scan.completed@1.sample.json @@ -70,7 +70,8 @@ "payload": "eyJyZXBvcnRJZCI6InJlcG9ydC1hYmMiLCJpbWFnZURpZ2VzdCI6InNoYTI1NjpmZWVkZmFjZSIsImdlbmVyYXRlZEF0IjoiMjAyNS0xMC0xOVQxMjozNDo1NiswMDowMCIsInZlcmRpY3QiOiJibG9ja2VkIiwicG9saWN5Ijp7InJldmlzaW9uSWQiOiJyZXYtNDIiLCJkaWdlc3QiOiJkaWdlc3QtMTIzIn0sInN1bW1hcnkiOnsidG90YWwiOjEsImJsb2NrZWQiOjEsIndhcm5lZCI6MCwiaWdub3JlZCI6MCwicXVpZXRlZCI6MH0sInZlcmRpY3RzIjpbeyJmaW5kaW5nSWQiOiJmaW5kaW5nLTEiLCJzdGF0dXMiOiJCbG9ja2VkIiwic2NvcmUiOjQ3LjUsInNvdXJjZVRydXN0IjoiTlZEIiwicmVhY2hhYmlsaXR5IjoicnVudGltZSJ9XSwiaXNzdWVzIjpbXX0=", "signatures": [ { - "keyId": "test-key", + "keyid": "test-key", + "sig": "", "algorithm": "hs256", "signature": "signature-value" }