stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

wip: doctor/cli/docs/api to vector db consolidation; api hardening for descriptions, tenant, and scopes; migrations and conversions of all DALs to EF v10

Browse Source
This commit is contained in:
master
2026-02-23 15:30:50 +02:00
parent bd8fee6ed8
commit e746577380
1424 changed files with 81225 additions and 25251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
src/Scanner/__Libraries/StellaOps.Scanner.SmartDiff/Detection/Repositories.cs
View File

@@ -11,22 +11,22 @@ public interface IRiskStateRepository
/// <summary>
/// Store a risk state snapshot.
/// </summary>
Task StoreSnapshotAsync(RiskStateSnapshot snapshot, CancellationToken ct = default);
Task StoreSnapshotAsync(RiskStateSnapshot snapshot, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Store multiple risk state snapshots.
/// </summary>
Task StoreSnapshotsAsync(IReadOnlyList<RiskStateSnapshot> snapshots, CancellationToken ct = default);
Task StoreSnapshotsAsync(IReadOnlyList<RiskStateSnapshot> snapshots, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Get the latest snapshot for a finding.
/// </summary>
Task<RiskStateSnapshot?> GetLatestSnapshotAsync(FindingKey findingKey, CancellationToken ct = default);
Task<RiskStateSnapshot?> GetLatestSnapshotAsync(FindingKey findingKey, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Get snapshots for a scan.
/// </summary>
Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsForScanAsync(string scanId, CancellationToken ct = default);
Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsForScanAsync(string scanId, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Get snapshot history for a finding.
@@ -34,12 +34,13 @@ public interface IRiskStateRepository
Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotHistoryAsync(
FindingKey findingKey,
int limit = 10,
CancellationToken ct = default);
CancellationToken ct = default,
string? tenantId = null);
/// <summary>
/// Get snapshots by state hash.
/// </summary>
Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsByHashAsync(string stateHash, CancellationToken ct = default);
Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsByHashAsync(string stateHash, CancellationToken ct = default, string? tenantId = null);
}
/// <summary>
@@ -50,17 +51,17 @@ public interface IMaterialRiskChangeRepository
/// <summary>
/// Store a material risk change result.
/// </summary>
Task StoreChangeAsync(MaterialRiskChangeResult change, string scanId, CancellationToken ct = default);
Task StoreChangeAsync(MaterialRiskChangeResult change, string scanId, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Store multiple material risk change results.
/// </summary>
Task StoreChangesAsync(IReadOnlyList<MaterialRiskChangeResult> changes, string scanId, CancellationToken ct = default);
Task StoreChangesAsync(IReadOnlyList<MaterialRiskChangeResult> changes, string scanId, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Get material changes for a scan.
/// </summary>
Task<IReadOnlyList<MaterialRiskChangeResult>> GetChangesForScanAsync(string scanId, CancellationToken ct = default);
Task<IReadOnlyList<MaterialRiskChangeResult>> GetChangesForScanAsync(string scanId, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Get material changes for a finding.
@@ -68,14 +69,16 @@ public interface IMaterialRiskChangeRepository
Task<IReadOnlyList<MaterialRiskChangeResult>> GetChangesForFindingAsync(
FindingKey findingKey,
int limit = 10,
CancellationToken ct = default);
CancellationToken ct = default,
string? tenantId = null);
/// <summary>
/// Query material changes with filters.
/// </summary>
Task<MaterialRiskChangeQueryResult> QueryChangesAsync(
MaterialRiskChangeQuery query,
CancellationToken ct = default);
CancellationToken ct = default,
string? tenantId = null);
}
/// <summary>
@@ -105,32 +108,40 @@ public sealed record MaterialRiskChangeQueryResult(
/// </summary>
public sealed class InMemoryRiskStateRepository : IRiskStateRepository
{
private readonly List<RiskStateSnapshot> _snapshots = [];
private readonly List<(string TenantId, RiskStateSnapshot Snapshot)> _snapshots = [];
private readonly object _lock = new();
public Task StoreSnapshotAsync(RiskStateSnapshot snapshot, CancellationToken ct = default)
public Task StoreSnapshotAsync(RiskStateSnapshot snapshot, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
lock (_lock)
{
_snapshots.Add(snapshot);
_snapshots.Add((normalizedTenant, snapshot));
}
return Task.CompletedTask;
}
public Task StoreSnapshotsAsync(IReadOnlyList<RiskStateSnapshot> snapshots, CancellationToken ct = default)
public Task StoreSnapshotsAsync(IReadOnlyList<RiskStateSnapshot> snapshots, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
lock (_lock)
{
_snapshots.AddRange(snapshots);
foreach (var snapshot in snapshots)
{
_snapshots.Add((normalizedTenant, snapshot));
}
}
return Task.CompletedTask;
}
public Task<RiskStateSnapshot?> GetLatestSnapshotAsync(FindingKey findingKey, CancellationToken ct = default)
public Task<RiskStateSnapshot?> GetLatestSnapshotAsync(FindingKey findingKey, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
lock (_lock)
{
var snapshot = _snapshots
.Where(entry => string.Equals(entry.TenantId, normalizedTenant, StringComparison.Ordinal))
.Select(entry => entry.Snapshot)
.Where(s => s.FindingKey == findingKey)
.OrderByDescending(s => s.CapturedAt)
.FirstOrDefault();
@@ -138,11 +149,14 @@ public sealed class InMemoryRiskStateRepository : IRiskStateRepository
}
}
public Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsForScanAsync(string scanId, CancellationToken ct = default)
public Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsForScanAsync(string scanId, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
lock (_lock)
{
var snapshots = _snapshots
.Where(entry => string.Equals(entry.TenantId, normalizedTenant, StringComparison.Ordinal))
.Select(entry => entry.Snapshot)
.Where(s => s.ScanId == scanId)
.ToList();
return Task.FromResult<IReadOnlyList<RiskStateSnapshot>>(snapshots);
@@ -152,11 +166,15 @@ public sealed class InMemoryRiskStateRepository : IRiskStateRepository
public Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotHistoryAsync(
FindingKey findingKey,
int limit = 10,
CancellationToken ct = default)
CancellationToken ct = default,
string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
lock (_lock)
{
var snapshots = _snapshots
.Where(entry => string.Equals(entry.TenantId, normalizedTenant, StringComparison.Ordinal))
.Select(entry => entry.Snapshot)
.Where(s => s.FindingKey == findingKey)
.OrderByDescending(s => s.CapturedAt)
.Take(limit)
@@ -165,16 +183,24 @@ public sealed class InMemoryRiskStateRepository : IRiskStateRepository
}
}
public Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsByHashAsync(string stateHash, CancellationToken ct = default)
public Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsByHashAsync(string stateHash, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
lock (_lock)
{
var snapshots = _snapshots
.Where(entry => string.Equals(entry.TenantId, normalizedTenant, StringComparison.Ordinal))
.Select(entry => entry.Snapshot)
.Where(s => s.ComputeStateHash() == stateHash)
.ToList();
return Task.FromResult<IReadOnlyList<RiskStateSnapshot>>(snapshots);
}
}
private static string NormalizeTenant(string? tenantId)
=> string.IsNullOrWhiteSpace(tenantId)
? "default"
: tenantId.Trim().ToLowerInvariant();
}
/// <summary>
@@ -186,54 +212,70 @@ public sealed class InMemoryVexCandidateStore : IVexCandidateStore
private readonly Dictionary<string, VexCandidateReview> _reviews = [];
private readonly object _lock = new();
public Task StoreCandidatesAsync(IReadOnlyList<VexCandidate> candidates, CancellationToken ct = default)
public Task StoreCandidatesAsync(IReadOnlyList<VexCandidate> candidates, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
lock (_lock)
{
foreach (var candidate in candidates)
{
_candidates[candidate.CandidateId] = candidate;
_candidates[BuildCandidateKey(normalizedTenant, candidate.CandidateId)] = candidate;
}
}
return Task.CompletedTask;
}
public Task<IReadOnlyList<VexCandidate>> GetCandidatesAsync(string imageDigest, CancellationToken ct = default)
public Task<IReadOnlyList<VexCandidate>> GetCandidatesAsync(string imageDigest, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
var tenantPrefix = $"{normalizedTenant}:";
lock (_lock)
{
var candidates = _candidates.Values
.Where(c => c.ImageDigest == imageDigest)
var candidates = _candidates
.Where(entry => entry.Key.StartsWith(tenantPrefix, StringComparison.Ordinal))
.Select(entry => entry.Value)
.Where(candidate => candidate.ImageDigest == imageDigest)
.ToList();
return Task.FromResult<IReadOnlyList<VexCandidate>>(candidates);
}
}
public Task<VexCandidate?> GetCandidateAsync(string candidateId, CancellationToken ct = default)
public Task<VexCandidate?> GetCandidateAsync(string candidateId, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
lock (_lock)
{
_candidates.TryGetValue(candidateId, out var candidate);
_candidates.TryGetValue(BuildCandidateKey(normalizedTenant, candidateId), out var candidate);
return Task.FromResult(candidate);
}
}
public Task<bool> ReviewCandidateAsync(string candidateId, VexCandidateReview review, CancellationToken ct = default)
public Task<bool> ReviewCandidateAsync(string candidateId, VexCandidateReview review, CancellationToken ct = default, string? tenantId = null)
{
var normalizedTenant = NormalizeTenant(tenantId);
var candidateKey = BuildCandidateKey(normalizedTenant, candidateId);
lock (_lock)
{
if (!_candidates.ContainsKey(candidateId))
if (!_candidates.ContainsKey(candidateKey))
return Task.FromResult(false);
_reviews[candidateId] = review;
_reviews[candidateKey] = review;
// Update candidate to mark as reviewed
if (_candidates.TryGetValue(candidateId, out var candidate))
if (_candidates.TryGetValue(candidateKey, out var candidate))
{
_candidates[candidateId] = candidate with { RequiresReview = false };
_candidates[candidateKey] = candidate with { RequiresReview = false };
}
return Task.FromResult(true);
}
}
private static string BuildCandidateKey(string tenantId, string candidateId)
=> $"{tenantId}:{candidateId}";
private static string NormalizeTenant(string? tenantId)
=> string.IsNullOrWhiteSpace(tenantId)
? "default"
: tenantId.Trim().ToLowerInvariant();
}

8
src/Scanner/__Libraries/StellaOps.Scanner.SmartDiff/Detection/VexCandidateModels.cs
View File

@@ -136,22 +136,22 @@ public interface IVexCandidateStore
/// <summary>
/// Store candidates.
/// </summary>
Task StoreCandidatesAsync(IReadOnlyList<VexCandidate> candidates, CancellationToken ct = default);
Task StoreCandidatesAsync(IReadOnlyList<VexCandidate> candidates, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Get candidates for an image.
/// </summary>
Task<IReadOnlyList<VexCandidate>> GetCandidatesAsync(string imageDigest, CancellationToken ct = default);
Task<IReadOnlyList<VexCandidate>> GetCandidatesAsync(string imageDigest, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Get a specific candidate by ID.
/// </summary>
Task<VexCandidate?> GetCandidateAsync(string candidateId, CancellationToken ct = default);
Task<VexCandidate?> GetCandidateAsync(string candidateId, CancellationToken ct = default, string? tenantId = null);
/// <summary>
/// Mark a candidate as reviewed.
/// </summary>
Task<bool> ReviewCandidateAsync(string candidateId, VexCandidateReview review, CancellationToken ct = default);
Task<bool> ReviewCandidateAsync(string candidateId, VexCandidateReview review, CancellationToken ct = default, string? tenantId = null);
}
/// <summary>

5
src/Scanner/__Libraries/StellaOps.Scanner.Sources/Persistence/ISbomSourceRepository.cs
View File

@@ -80,12 +80,13 @@ public interface ISbomSourceRunRepository
/// <summary>
/// Get a run by ID.
/// </summary>
Task<SbomSourceRun?> GetByIdAsync(Guid runId, CancellationToken ct = default);
Task<SbomSourceRun?> GetByIdAsync(string tenantId, Guid runId, CancellationToken ct = default);
/// <summary>
/// List runs for a source.
/// </summary>
Task<PagedResponse<SbomSourceRun>> ListForSourceAsync(
string tenantId,
Guid sourceId,
ListSourceRunsRequest request,
CancellationToken ct = default);
@@ -111,7 +112,7 @@ public interface ISbomSourceRunRepository
/// <summary>
/// Get aggregate statistics for a source.
/// </summary>
Task<SourceRunStats> GetStatsAsync(Guid sourceId, CancellationToken ct = default);
Task<SourceRunStats> GetStatsAsync(string tenantId, Guid sourceId, CancellationToken ct = default);
}
/// <summary>

36
src/Scanner/__Libraries/StellaOps.Scanner.Sources/Persistence/SbomSourceRunRepository.cs
View File

@@ -28,32 +28,37 @@ public sealed class SbomSourceRunRepository : RepositoryBase<ScannerSourcesDataS
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<SbomSourceRun?> GetByIdAsync(Guid runId, CancellationToken ct = default)
public async Task<SbomSourceRun?> GetByIdAsync(string tenantId, Guid runId, CancellationToken ct = default)
{
const string sql = $"""
SELECT * FROM {FullTable}
WHERE run_id = @runId
WHERE tenant_id = @tenantId AND run_id = @runId
""";
// Use system tenant for run queries (runs have their own tenant_id)
return await QuerySingleOrDefaultAsync(
"__system__",
tenantId,
sql,
cmd => AddParameter(cmd, "runId", runId),
cmd =>
{
AddParameter(cmd, "tenantId", tenantId);
AddParameter(cmd, "runId", runId);
},
MapRun,
ct);
}
public async Task<PagedResponse<SbomSourceRun>> ListForSourceAsync(
string tenantId,
Guid sourceId,
ListSourceRunsRequest request,
CancellationToken ct = default)
{
var sb = new StringBuilder($"SELECT * FROM {FullTable} WHERE source_id = @sourceId");
var countSb = new StringBuilder($"SELECT COUNT(*) FROM {FullTable} WHERE source_id = @sourceId");
var sb = new StringBuilder($"SELECT * FROM {FullTable} WHERE tenant_id = @tenantId AND source_id = @sourceId");
var countSb = new StringBuilder($"SELECT COUNT(*) FROM {FullTable} WHERE tenant_id = @tenantId AND source_id = @sourceId");
void AddFilters(NpgsqlCommand cmd)
{
AddParameter(cmd, "tenantId", tenantId);
AddParameter(cmd, "sourceId", sourceId);
if (request.Trigger.HasValue)
@@ -95,14 +100,14 @@ public sealed class SbomSourceRunRepository : RepositoryBase<ScannerSourcesDataS
}
var items = await QueryAsync(
"__system__",
tenantId,
sb.ToString(),
AddFilters,
MapRun,
ct);
var totalCount = await ExecuteScalarAsync<long>(
"__system__",
tenantId,
countSb.ToString(),
AddFilters,
ct);
@@ -197,7 +202,7 @@ public sealed class SbomSourceRunRepository : RepositoryBase<ScannerSourcesDataS
ct);
}
public async Task<SourceRunStats> GetStatsAsync(Guid sourceId, CancellationToken ct = default)
public async Task<SourceRunStats> GetStatsAsync(string tenantId, Guid sourceId, CancellationToken ct = default)
{
const string sql = $"""
SELECT
@@ -209,14 +214,19 @@ public sealed class SbomSourceRunRepository : RepositoryBase<ScannerSourcesDataS
MAX(completed_at) FILTER (WHERE status = 'Succeeded') as last_success_at,
MAX(completed_at) FILTER (WHERE status = 'Failed') as last_failure_at
FROM {FullTable}
WHERE source_id = @sourceId
WHERE tenant_id = @tenantId
AND source_id = @sourceId
AND completed_at IS NOT NULL
""";
var result = await QuerySingleOrDefaultAsync(
"__system__",
tenantId,
sql,
cmd => AddParameter(cmd, "sourceId", sourceId),
cmd =>
{
AddParameter(cmd, "tenantId", tenantId);
AddParameter(cmd, "sourceId", sourceId);
},
reader => new SourceRunStats
{
TotalRuns = reader.GetInt32(reader.GetOrdinal("total_runs")),

4
src/Scanner/__Libraries/StellaOps.Scanner.Sources/Services/SbomSourceService.cs
View File

@@ -379,7 +379,7 @@ public sealed class SbomSourceService : ISbomSourceService
var source = await _sourceRepository.GetByIdAsync(tenantId, sourceId, ct)
?? throw new KeyNotFoundException($"Source {sourceId} not found");
var result = await _runRepository.ListForSourceAsync(sourceId, request, ct);
var result = await _runRepository.ListForSourceAsync(tenantId, sourceId, request, ct);
return new PagedResponse<SourceRunResponse>
{
@@ -399,7 +399,7 @@ public sealed class SbomSourceService : ISbomSourceService
_ = await _sourceRepository.GetByIdAsync(tenantId, sourceId, ct)
?? throw new KeyNotFoundException($"Source {sourceId} not found");
var run = await _runRepository.GetByIdAsync(runId, ct);
var run = await _runRepository.GetByIdAsync(tenantId, runId, ct);
if (run == null || run.SourceId != sourceId)
{
return null;

1
src/Scanner/__Libraries/StellaOps.Scanner.Sources/TASKS.md
View File

@@ -9,3 +9,4 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229
| AUDIT-0684-T | DONE | Revalidated 2026-01-12. |
| AUDIT-0684-A | DONE | Applied 2026-01-14. |
| REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |
| SPRINT-20260222-057-SCAN-TEN-13 | DONE | `SPRINT_20260222_057_Scanner_tenant_isolation_for_scans_triage_webhooks.md`: tenant-parameterized `ISbomSourceRunRepository` (`GetByIdAsync`, `ListForSourceAsync`, `GetStatsAsync`) and SQL predicates for `scanner.sbom_source_runs` (2026-02-23). |

8
src/Scanner/__Libraries/StellaOps.Scanner.Sources/Triggers/SourceTriggerDispatcher.cs
View File

@@ -248,7 +248,13 @@ public sealed class SourceTriggerDispatcher : ISourceTriggerDispatcher
Guid originalRunId,
CancellationToken ct = default)
{
var originalRun = await _runRepository.GetByIdAsync(originalRunId, ct);
var source = await _sourceRepository.GetByIdAnyTenantAsync(sourceId, ct);
if (source == null)
{
throw new KeyNotFoundException($"Source {sourceId} not found");
}
var originalRun = await _runRepository.GetByIdAsync(source.TenantId, originalRunId, ct);
if (originalRun == null)
{
throw new KeyNotFoundException($"Run {originalRunId} not found");

9
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/CompiledModels/ScannerDbContextAssemblyAttributes.cs Normal file
View File

@@ -0,0 +1,9 @@
// <auto-generated />
using Microsoft.EntityFrameworkCore.Infrastructure;
using StellaOps.Scanner.Storage.EfCore.CompiledModels;
using StellaOps.Scanner.Storage.EfCore.Context;
#pragma warning disable 219, 612, 618
#nullable disable
[assembly: DbContextModel(typeof(ScannerDbContext), typeof(ScannerDbContextModel))]

48
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/CompiledModels/ScannerDbContextModel.cs Normal file
View File

@@ -0,0 +1,48 @@
// <auto-generated />
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using StellaOps.Scanner.Storage.EfCore.Context;
#pragma warning disable 219, 612, 618
#nullable disable
namespace StellaOps.Scanner.Storage.EfCore.CompiledModels
{
[DbContext(typeof(ScannerDbContext))]
public partial class ScannerDbContextModel : RuntimeModel
{
private static readonly bool _useOldBehavior31751 =
System.AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue31751", out var enabled31751) && enabled31751;
static ScannerDbContextModel()
{
var model = new ScannerDbContextModel();
if (_useOldBehavior31751)
{
model.Initialize();
}
else
{
var thread = new System.Threading.Thread(RunInitialization, 10 * 1024 * 1024);
thread.Start();
thread.Join();
void RunInitialization()
{
model.Initialize();
}
}
model.Customize();
_instance = (ScannerDbContextModel)model.FinalizeModel();
}
private static ScannerDbContextModel _instance;
public static IModel Instance => _instance;
partial void Initialize();
partial void Customize();
}
}

27
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/CompiledModels/ScannerDbContextModelBuilder.cs Normal file
View File

@@ -0,0 +1,27 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#pragma warning disable 219, 612, 618
#nullable disable
namespace StellaOps.Scanner.Storage.EfCore.CompiledModels
{
public partial class ScannerDbContextModel
{
private ScannerDbContextModel()
: base(skipDetectChanges: false, modelId: new Guid("b2a4e1c7-8d3f-4a5b-9e6c-1f7d2e8b3c4a"), entityTypeCount: 13)
{
}
partial void Initialize()
{
// Stub: entity types will be populated by `dotnet ef dbcontext optimize`.
AddAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn);
AddAnnotation("ProductVersion", "10.0.0");
AddAnnotation("Relational:MaxIdentifierLength", 63);
}
}
}

397
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Context/ScannerDbContext.cs Normal file
View File

@@ -0,0 +1,397 @@
using Microsoft.EntityFrameworkCore;
using StellaOps.Scanner.Storage.EfCore.Models;
namespace StellaOps.Scanner.Storage.EfCore.Context;
/// <summary>
/// Entity Framework Core DbContext for the Scanner Storage schema.
/// SQL migrations remain authoritative; EF models are scaffolded FROM schema.
/// </summary>
public partial class ScannerDbContext : DbContext
{
private readonly string _schemaName;
public ScannerDbContext(DbContextOptions<ScannerDbContext> options, string? schemaName = null)
: base(options)
{
_schemaName = string.IsNullOrWhiteSpace(schemaName)
? ScannerStorageDefaults.DefaultSchemaName
: schemaName.Trim();
}
// ----- Scanner schema tables -----
public virtual DbSet<IdempotencyKeyEntity> IdempotencyKeys { get; set; }
public virtual DbSet<ScanMetricsEntity> ScanMetrics { get; set; }
public virtual DbSet<RiskStateSnapshotEntity> RiskStateSnapshots { get; set; }
public virtual DbSet<MaterialRiskChangeEntity> MaterialRiskChanges { get; set; }
public virtual DbSet<CallGraphSnapshotEntity> CallGraphSnapshots { get; set; }
public virtual DbSet<ReachabilityResultEntity> ReachabilityResults { get; set; }
// ----- Public/default schema tables -----
public virtual DbSet<ScanManifestEntity> ScanManifests { get; set; }
public virtual DbSet<ProofBundleEntity> ProofBundles { get; set; }
public virtual DbSet<BinaryIdentityEntity> BinaryIdentities { get; set; }
public virtual DbSet<BinaryPackageMapEntity> BinaryPackageMaps { get; set; }
public virtual DbSet<BinaryVulnAssertionEntity> BinaryVulnAssertions { get; set; }
public virtual DbSet<SecretDetectionSettingsEntity> SecretDetectionSettings { get; set; }
public virtual DbSet<ArtifactBomEntity> ArtifactBoms { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
var schema = _schemaName;
// ======================================================================
// Scanner schema tables
// ======================================================================
modelBuilder.Entity<IdempotencyKeyEntity>(entity =>
{
entity.ToTable("idempotency_keys", schema);
entity.HasKey(e => e.KeyId);
entity.Property(e => e.KeyId).HasColumnName("key_id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.TenantId).HasColumnName("tenant_id");
entity.Property(e => e.ContentDigest).HasColumnName("content_digest");
entity.Property(e => e.EndpointPath).HasColumnName("endpoint_path");
entity.Property(e => e.ResponseStatus).HasColumnName("response_status");
entity.Property(e => e.ResponseBody).HasColumnName("response_body").HasColumnType("jsonb");
entity.Property(e => e.ResponseHeaders).HasColumnName("response_headers").HasColumnType("jsonb");
entity.Property(e => e.CreatedAt).HasColumnName("created_at").HasDefaultValueSql("now()");
entity.Property(e => e.ExpiresAt).HasColumnName("expires_at").HasDefaultValueSql("(now() + interval '24 hours')");
entity.HasIndex(e => new { e.TenantId, e.ContentDigest, e.EndpointPath })
.IsUnique()
.HasDatabaseName("uk_idempotency_tenant_digest_path");
entity.HasIndex(e => new { e.TenantId, e.ContentDigest })
.HasDatabaseName("ix_idempotency_keys_tenant_digest");
entity.HasIndex(e => e.ExpiresAt)
.HasDatabaseName("ix_idempotency_keys_expires_at");
});
modelBuilder.Entity<ScanMetricsEntity>(entity =>
{
entity.ToTable("scan_metrics", schema);
entity.HasKey(e => e.MetricsId);
entity.Property(e => e.MetricsId).HasColumnName("metrics_id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.ScanId).HasColumnName("scan_id");
entity.Property(e => e.TenantId).HasColumnName("tenant_id");
entity.Property(e => e.SurfaceId).HasColumnName("surface_id");
entity.Property(e => e.ArtifactDigest).HasColumnName("artifact_digest");
entity.Property(e => e.ArtifactType).HasColumnName("artifact_type");
entity.Property(e => e.ReplayManifestHash).HasColumnName("replay_manifest_hash");
entity.Property(e => e.FindingsSha256).HasColumnName("findings_sha256");
entity.Property(e => e.VexBundleSha256).HasColumnName("vex_bundle_sha256");
entity.Property(e => e.ProofBundleSha256).HasColumnName("proof_bundle_sha256");
entity.Property(e => e.SbomSha256).HasColumnName("sbom_sha256");
entity.Property(e => e.PolicyDigest).HasColumnName("policy_digest");
entity.Property(e => e.FeedSnapshotId).HasColumnName("feed_snapshot_id");
entity.Property(e => e.StartedAt).HasColumnName("started_at");
entity.Property(e => e.FinishedAt).HasColumnName("finished_at");
entity.Property(e => e.TIngestMs).HasColumnName("t_ingest_ms");
entity.Property(e => e.TAnalyzeMs).HasColumnName("t_analyze_ms");
entity.Property(e => e.TReachabilityMs).HasColumnName("t_reachability_ms");
entity.Property(e => e.TVexMs).HasColumnName("t_vex_ms");
entity.Property(e => e.TSignMs).HasColumnName("t_sign_ms");
entity.Property(e => e.TPublishMs).HasColumnName("t_publish_ms");
entity.Property(e => e.PackageCount).HasColumnName("package_count");
entity.Property(e => e.FindingCount).HasColumnName("finding_count");
entity.Property(e => e.VexDecisionCount).HasColumnName("vex_decision_count");
entity.Property(e => e.ScannerVersion).HasColumnName("scanner_version");
entity.Property(e => e.ScannerImageDigest).HasColumnName("scanner_image_digest");
entity.Property(e => e.IsReplay).HasColumnName("is_replay");
entity.Property(e => e.CreatedAt).HasColumnName("created_at").HasDefaultValueSql("NOW()");
entity.HasIndex(e => e.ScanId).IsUnique().HasDatabaseName("scan_metrics_scan_id_key");
entity.HasIndex(e => e.TenantId).HasDatabaseName("idx_scan_metrics_tenant");
entity.HasIndex(e => e.ArtifactDigest).HasDatabaseName("idx_scan_metrics_artifact");
entity.HasIndex(e => e.StartedAt).HasDatabaseName("idx_scan_metrics_started");
entity.HasIndex(e => new { e.TenantId, e.StartedAt }).HasDatabaseName("idx_scan_metrics_tenant_started");
});
modelBuilder.Entity<RiskStateSnapshotEntity>(entity =>
{
entity.ToTable("risk_state_snapshots", schema);
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.TenantId).HasColumnName("tenant_id");
entity.Property(e => e.VulnId).HasColumnName("vuln_id");
entity.Property(e => e.Purl).HasColumnName("purl");
entity.Property(e => e.ScanId).HasColumnName("scan_id");
entity.Property(e => e.CapturedAt).HasColumnName("captured_at").HasDefaultValueSql("NOW()");
entity.Property(e => e.Reachable).HasColumnName("reachable");
entity.Property(e => e.LatticeState).HasColumnName("lattice_state");
entity.Property(e => e.VexStatus).HasColumnName("vex_status");
entity.Property(e => e.InAffectedRange).HasColumnName("in_affected_range");
entity.Property(e => e.Kev).HasColumnName("kev");
entity.Property(e => e.EpssScore).HasColumnName("epss_score").HasColumnType("numeric(5,4)");
entity.Property(e => e.PolicyFlags).HasColumnName("policy_flags");
entity.Property(e => e.PolicyDecision).HasColumnName("policy_decision");
entity.Property(e => e.StateHash).HasColumnName("state_hash");
entity.Property(e => e.CreatedAt).HasColumnName("created_at").HasDefaultValueSql("NOW()");
entity.HasIndex(e => new { e.TenantId, e.ScanId, e.VulnId, e.Purl })
.IsUnique()
.HasDatabaseName("risk_state_unique_per_scan");
entity.HasIndex(e => new { e.TenantId, e.VulnId, e.Purl })
.HasDatabaseName("idx_risk_state_tenant_finding");
entity.HasIndex(e => e.ScanId).HasDatabaseName("idx_risk_state_scan");
entity.HasIndex(e => e.StateHash).HasDatabaseName("idx_risk_state_hash");
});
modelBuilder.Entity<MaterialRiskChangeEntity>(entity =>
{
entity.ToTable("material_risk_changes", schema);
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.TenantId).HasColumnName("tenant_id");
entity.Property(e => e.VulnId).HasColumnName("vuln_id");
entity.Property(e => e.Purl).HasColumnName("purl");
entity.Property(e => e.ScanId).HasColumnName("scan_id");
entity.Property(e => e.HasMaterialChange).HasColumnName("has_material_change");
entity.Property(e => e.PriorityScore).HasColumnName("priority_score").HasColumnType("numeric(12,4)");
entity.Property(e => e.PreviousStateHash).HasColumnName("previous_state_hash");
entity.Property(e => e.CurrentStateHash).HasColumnName("current_state_hash");
entity.Property(e => e.Changes).HasColumnName("changes").HasColumnType("jsonb");
entity.Property(e => e.DetectedAt).HasColumnName("detected_at").HasDefaultValueSql("NOW()");
entity.Property(e => e.BaseScanId).HasColumnName("base_scan_id");
entity.Property(e => e.Cause).HasColumnName("cause");
entity.Property(e => e.CauseKind).HasColumnName("cause_kind");
entity.Property(e => e.PathNodes).HasColumnName("path_nodes").HasColumnType("jsonb");
entity.Property(e => e.AssociatedVulns).HasColumnName("associated_vulns").HasColumnType("jsonb");
entity.HasIndex(e => new { e.TenantId, e.ScanId, e.VulnId, e.Purl })
.IsUnique()
.HasDatabaseName("material_change_unique_per_scan");
entity.HasIndex(e => new { e.TenantId, e.ScanId })
.HasDatabaseName("idx_material_changes_tenant_scan");
});
modelBuilder.Entity<CallGraphSnapshotEntity>(entity =>
{
entity.ToTable("call_graph_snapshots");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.TenantId).HasColumnName("tenant_id");
entity.Property(e => e.ScanId).HasColumnName("scan_id");
entity.Property(e => e.Language).HasColumnName("language");
entity.Property(e => e.GraphDigest).HasColumnName("graph_digest");
entity.Property(e => e.ExtractedAt).HasColumnName("extracted_at").HasDefaultValueSql("NOW()");
entity.Property(e => e.NodeCount).HasColumnName("node_count");
entity.Property(e => e.EdgeCount).HasColumnName("edge_count");
entity.Property(e => e.EntrypointCount).HasColumnName("entrypoint_count");
entity.Property(e => e.SinkCount).HasColumnName("sink_count");
entity.Property(e => e.SnapshotJson).HasColumnName("snapshot_json").HasColumnType("jsonb");
entity.HasIndex(e => new { e.TenantId, e.ScanId, e.Language, e.GraphDigest })
.IsUnique()
.HasDatabaseName("call_graph_snapshot_unique_per_scan");
entity.HasIndex(e => new { e.TenantId, e.ScanId, e.Language })
.HasDatabaseName("idx_call_graph_snapshots_tenant_scan");
entity.HasIndex(e => e.GraphDigest).HasDatabaseName("idx_call_graph_snapshots_graph_digest");
});
modelBuilder.Entity<ReachabilityResultEntity>(entity =>
{
entity.ToTable("reachability_results");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.TenantId).HasColumnName("tenant_id");
entity.Property(e => e.ScanId).HasColumnName("scan_id");
entity.Property(e => e.Language).HasColumnName("language");
entity.Property(e => e.GraphDigest).HasColumnName("graph_digest");
entity.Property(e => e.ResultDigest).HasColumnName("result_digest");
entity.Property(e => e.ComputedAt).HasColumnName("computed_at").HasDefaultValueSql("NOW()");
entity.Property(e => e.ReachableNodeCount).HasColumnName("reachable_node_count");
entity.Property(e => e.ReachableSinkCount).HasColumnName("reachable_sink_count");
entity.Property(e => e.ResultJson).HasColumnName("result_json").HasColumnType("jsonb");
entity.HasIndex(e => new { e.TenantId, e.ScanId, e.Language, e.GraphDigest, e.ResultDigest })
.IsUnique()
.HasDatabaseName("reachability_result_unique_per_scan");
entity.HasIndex(e => new { e.TenantId, e.ScanId, e.Language })
.HasDatabaseName("idx_reachability_results_tenant_scan");
});
// ======================================================================
// Public/default schema tables
// ======================================================================
modelBuilder.Entity<ScanManifestEntity>(entity =>
{
entity.ToTable("scan_manifest");
entity.HasKey(e => e.ManifestId);
entity.Property(e => e.ManifestId).HasColumnName("manifest_id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.ScanId).HasColumnName("scan_id");
entity.Property(e => e.ManifestHash).HasColumnName("manifest_hash").HasMaxLength(128);
entity.Property(e => e.SbomHash).HasColumnName("sbom_hash").HasMaxLength(128);
entity.Property(e => e.RulesHash).HasColumnName("rules_hash").HasMaxLength(128);
entity.Property(e => e.FeedHash).HasColumnName("feed_hash").HasMaxLength(128);
entity.Property(e => e.PolicyHash).HasColumnName("policy_hash").HasMaxLength(128);
entity.Property(e => e.ScanStartedAt).HasColumnName("scan_started_at");
entity.Property(e => e.ScanCompletedAt).HasColumnName("scan_completed_at");
entity.Property(e => e.ManifestContent).HasColumnName("manifest_content").HasColumnType("jsonb");
entity.Property(e => e.ScannerVersion).HasColumnName("scanner_version").HasMaxLength(64);
entity.Property(e => e.CreatedAt).HasColumnName("created_at").HasDefaultValueSql("now()");
entity.HasIndex(e => e.ManifestHash).HasDatabaseName("idx_scan_manifest_hash");
entity.HasIndex(e => e.ScanId).HasDatabaseName("idx_scan_manifest_scan_id");
entity.HasIndex(e => e.CreatedAt).IsDescending().HasDatabaseName("idx_scan_manifest_created_at");
});
modelBuilder.Entity<ProofBundleEntity>(entity =>
{
entity.ToTable("proof_bundle");
entity.HasKey(e => new { e.ScanId, e.RootHash });
entity.Property(e => e.ScanId).HasColumnName("scan_id");
entity.Property(e => e.RootHash).HasColumnName("root_hash").HasMaxLength(128);
entity.Property(e => e.BundleType).HasColumnName("bundle_type").HasMaxLength(32);
entity.Property(e => e.DsseEnvelope).HasColumnName("dsse_envelope").HasColumnType("jsonb");
entity.Property(e => e.SignatureKeyId).HasColumnName("signature_keyid").HasMaxLength(256);
entity.Property(e => e.SignatureAlgorithm).HasColumnName("signature_algorithm").HasMaxLength(64);
entity.Property(e => e.BundleContent).HasColumnName("bundle_content");
entity.Property(e => e.BundleHash).HasColumnName("bundle_hash").HasMaxLength(128);
entity.Property(e => e.LedgerHash).HasColumnName("ledger_hash").HasMaxLength(128);
entity.Property(e => e.ManifestHash).HasColumnName("manifest_hash").HasMaxLength(128);
entity.Property(e => e.SbomHash).HasColumnName("sbom_hash").HasMaxLength(128);
entity.Property(e => e.VexHash).HasColumnName("vex_hash").HasMaxLength(128);
entity.Property(e => e.CreatedAt).HasColumnName("created_at").HasDefaultValueSql("now()");
entity.Property(e => e.ExpiresAt).HasColumnName("expires_at");
entity.HasIndex(e => e.RootHash).HasDatabaseName("idx_proof_bundle_root_hash");
entity.HasIndex(e => e.CreatedAt).IsDescending().HasDatabaseName("idx_proof_bundle_created_at");
});
modelBuilder.Entity<BinaryIdentityEntity>(entity =>
{
entity.ToTable("binary_identity");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.ScanId).HasColumnName("scan_id");
entity.Property(e => e.FilePath).HasColumnName("file_path").HasMaxLength(1024);
entity.Property(e => e.FileSha256).HasColumnName("file_sha256").HasMaxLength(64);
entity.Property(e => e.TextSha256).HasColumnName("text_sha256").HasMaxLength(64);
entity.Property(e => e.BuildId).HasColumnName("build_id").HasMaxLength(128);
entity.Property(e => e.BuildIdType).HasColumnName("build_id_type").HasMaxLength(32);
entity.Property(e => e.Architecture).HasColumnName("architecture").HasMaxLength(32);
entity.Property(e => e.BinaryFormat).HasColumnName("binary_format").HasMaxLength(16);
entity.Property(e => e.FileSize).HasColumnName("file_size");
entity.Property(e => e.IsStripped).HasColumnName("is_stripped");
entity.Property(e => e.HasDebugInfo).HasColumnName("has_debug_info");
entity.Property(e => e.CreatedAtUtc).HasColumnName("created_at_utc").HasDefaultValueSql("NOW()");
entity.HasIndex(e => e.BuildId).HasDatabaseName("idx_binary_identity_build_id");
entity.HasIndex(e => e.FileSha256).HasDatabaseName("idx_binary_identity_file_sha256");
entity.HasIndex(e => e.TextSha256).HasDatabaseName("idx_binary_identity_text_sha256");
entity.HasIndex(e => e.ScanId).HasDatabaseName("idx_binary_identity_scan_id");
});
modelBuilder.Entity<BinaryPackageMapEntity>(entity =>
{
entity.ToTable("binary_package_map");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.BinaryIdentityId).HasColumnName("binary_identity_id");
entity.Property(e => e.Purl).HasColumnName("purl").HasMaxLength(512);
entity.Property(e => e.MatchType).HasColumnName("match_type").HasMaxLength(32);
entity.Property(e => e.Confidence).HasColumnName("confidence").HasColumnType("numeric(3,2)");
entity.Property(e => e.MatchSource).HasColumnName("match_source").HasMaxLength(64);
entity.Property(e => e.EvidenceJson).HasColumnName("evidence_json").HasColumnType("jsonb");
entity.Property(e => e.CreatedAtUtc).HasColumnName("created_at_utc").HasDefaultValueSql("NOW()");
entity.HasIndex(e => new { e.BinaryIdentityId, e.Purl }).IsUnique().HasDatabaseName("uq_binary_package_map");
entity.HasIndex(e => e.Purl).HasDatabaseName("idx_binary_package_map_purl");
entity.HasIndex(e => e.BinaryIdentityId).HasDatabaseName("idx_binary_package_map_binary_id");
entity.HasOne(e => e.BinaryIdentity)
.WithMany(b => b.PackageMaps)
.HasForeignKey(e => e.BinaryIdentityId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<BinaryVulnAssertionEntity>(entity =>
{
entity.ToTable("binary_vuln_assertion");
entity.HasKey(e => e.Id);
entity.Property(e => e.Id).HasColumnName("id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.BinaryIdentityId).HasColumnName("binary_identity_id");
entity.Property(e => e.VulnId).HasColumnName("vuln_id").HasMaxLength(64);
entity.Property(e => e.Status).HasColumnName("status").HasMaxLength(32);
entity.Property(e => e.Source).HasColumnName("source").HasMaxLength(64);
entity.Property(e => e.AssertionType).HasColumnName("assertion_type").HasMaxLength(32);
entity.Property(e => e.Confidence).HasColumnName("confidence").HasColumnType("numeric(3,2)");
entity.Property(e => e.EvidenceJson).HasColumnName("evidence_json").HasColumnType("jsonb");
entity.Property(e => e.ValidFrom).HasColumnName("valid_from");
entity.Property(e => e.ValidUntil).HasColumnName("valid_until");
entity.Property(e => e.SignatureRef).HasColumnName("signature_ref").HasMaxLength(256);
entity.Property(e => e.CreatedAtUtc).HasColumnName("created_at_utc").HasDefaultValueSql("NOW()");
entity.HasIndex(e => e.VulnId).HasDatabaseName("idx_binary_vuln_assertion_vuln_id");
entity.HasIndex(e => e.BinaryIdentityId).HasDatabaseName("idx_binary_vuln_assertion_binary_id");
entity.HasIndex(e => e.Status).HasDatabaseName("idx_binary_vuln_assertion_status");
entity.HasOne(e => e.BinaryIdentity)
.WithMany(b => b.VulnAssertions)
.HasForeignKey(e => e.BinaryIdentityId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<SecretDetectionSettingsEntity>(entity =>
{
entity.ToTable("secret_detection_settings", schema);
entity.HasKey(e => e.SettingsId);
entity.Property(e => e.SettingsId).HasColumnName("settings_id").HasDefaultValueSql("gen_random_uuid()");
entity.Property(e => e.TenantId).HasColumnName("tenant_id");
entity.Property(e => e.Enabled).HasColumnName("enabled");
entity.Property(e => e.RevelationPolicy).HasColumnName("revelation_policy").HasColumnType("jsonb");
entity.Property(e => e.EnabledRuleCategories).HasColumnName("enabled_rule_categories");
entity.Property(e => e.DisabledRuleIds).HasColumnName("disabled_rule_ids");
entity.Property(e => e.AlertSettings).HasColumnName("alert_settings").HasColumnType("jsonb");
entity.Property(e => e.MaxFileSizeBytes).HasColumnName("max_file_size_bytes");
entity.Property(e => e.ExcludedFileExtensions).HasColumnName("excluded_file_extensions");
entity.Property(e => e.ExcludedPaths).HasColumnName("excluded_paths");
entity.Property(e => e.ScanBinaryFiles).HasColumnName("scan_binary_files");
entity.Property(e => e.RequireSignedRuleBundles).HasColumnName("require_signed_rule_bundles");
entity.Property(e => e.Version).HasColumnName("version");
entity.Property(e => e.UpdatedAt).HasColumnName("updated_at").HasDefaultValueSql("NOW()");
entity.Property(e => e.UpdatedBy).HasColumnName("updated_by");
entity.Property(e => e.CreatedAt).HasColumnName("created_at").HasDefaultValueSql("NOW()");
entity.HasIndex(e => e.TenantId).IsUnique().HasDatabaseName("secret_detection_settings_tenant_id_key");
});
modelBuilder.Entity<ArtifactBomEntity>(entity =>
{
entity.ToTable("artifact_boms", schema);
entity.HasKey(e => new { e.BuildId, e.InsertedAt });
entity.Property(e => e.BuildId).HasColumnName("build_id");
entity.Property(e => e.CanonicalBomSha256).HasColumnName("canonical_bom_sha256");
entity.Property(e => e.PayloadDigest).HasColumnName("payload_digest");
entity.Property(e => e.InsertedAt).HasColumnName("inserted_at");
entity.Property(e => e.RawBomRef).HasColumnName("raw_bom_ref");
entity.Property(e => e.CanonicalBomRef).HasColumnName("canonical_bom_ref");
entity.Property(e => e.DsseEnvelopeRef).HasColumnName("dsse_envelope_ref");
entity.Property(e => e.MergedVexRef).HasColumnName("merged_vex_ref");
entity.Property(e => e.CanonicalBomJson).HasColumnName("canonical_bom").HasColumnType("jsonb");
entity.Property(e => e.MergedVexJson).HasColumnName("merged_vex").HasColumnType("jsonb");
entity.Property(e => e.AttestationsJson).HasColumnName("attestations").HasColumnType("jsonb");
entity.Property(e => e.EvidenceScore).HasColumnName("evidence_score");
entity.Property(e => e.RekorTileId).HasColumnName("rekor_tile_id");
});
OnModelCreatingPartial(modelBuilder);
}
partial void OnModelCreatingPartial(ModelBuilder modelBuilder);
}

29
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Context/ScannerDesignTimeDbContextFactory.cs Normal file
View File

@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
namespace StellaOps.Scanner.Storage.EfCore.Context;
/// <summary>
/// Design-time factory for EF Core tooling (scaffold, optimize, migrations).
/// </summary>
public sealed class ScannerDesignTimeDbContextFactory : IDesignTimeDbContextFactory<ScannerDbContext>
{
private const string DefaultConnectionString = "Host=localhost;Port=55434;Database=postgres;Username=postgres;Password=postgres;Search Path=scanner,public";
private const string ConnectionStringEnvironmentVariable = "STELLAOPS_SCANNER_EF_CONNECTION";
public ScannerDbContext CreateDbContext(string[] args)
{
var connectionString = ResolveConnectionString();
var options = new DbContextOptionsBuilder<ScannerDbContext>()
.UseNpgsql(connectionString)
.Options;
return new ScannerDbContext(options);
}
private static string ResolveConnectionString()
{
var fromEnvironment = Environment.GetEnvironmentVariable(ConnectionStringEnvironmentVariable);
return string.IsNullOrWhiteSpace(fromEnvironment) ? DefaultConnectionString : fromEnvironment;
}
}

21
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/ArtifactBomEntity.cs Normal file
View File

@@ -0,0 +1,21 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the artifact_boms partitioned table.
/// </summary>
public sealed class ArtifactBomEntity
{
public string BuildId { get; set; } = null!;
public string CanonicalBomSha256 { get; set; } = null!;
public string PayloadDigest { get; set; } = null!;
public DateTimeOffset InsertedAt { get; set; }
public string? RawBomRef { get; set; }
public string? CanonicalBomRef { get; set; }
public string? DsseEnvelopeRef { get; set; }
public string? MergedVexRef { get; set; }
public string? CanonicalBomJson { get; set; }
public string? MergedVexJson { get; set; }
public string? AttestationsJson { get; set; }
public double? EvidenceScore { get; set; }
public string? RekorTileId { get; set; }
}

24
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/BinaryIdentityEntity.cs Normal file
View File

@@ -0,0 +1,24 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the binary_identity table.
/// </summary>
public sealed class BinaryIdentityEntity
{
public Guid Id { get; set; }
public Guid ScanId { get; set; }
public string FilePath { get; set; } = null!;
public string FileSha256 { get; set; } = null!;
public string? TextSha256 { get; set; }
public string? BuildId { get; set; }
public string? BuildIdType { get; set; }
public string Architecture { get; set; } = null!;
public string BinaryFormat { get; set; } = null!;
public long FileSize { get; set; }
public bool IsStripped { get; set; }
public bool HasDebugInfo { get; set; }
public DateTimeOffset CreatedAtUtc { get; set; }
public ICollection<BinaryPackageMapEntity> PackageMaps { get; set; } = new List<BinaryPackageMapEntity>();
public ICollection<BinaryVulnAssertionEntity> VulnAssertions { get; set; } = new List<BinaryVulnAssertionEntity>();
}

18
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/BinaryPackageMapEntity.cs Normal file
View File

@@ -0,0 +1,18 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the binary_package_map table.
/// </summary>
public sealed class BinaryPackageMapEntity
{
public Guid Id { get; set; }
public Guid BinaryIdentityId { get; set; }
public string Purl { get; set; } = null!;
public string MatchType { get; set; } = null!;
public decimal Confidence { get; set; }
public string MatchSource { get; set; } = null!;
public string? EvidenceJson { get; set; }
public DateTimeOffset CreatedAtUtc { get; set; }
public BinaryIdentityEntity? BinaryIdentity { get; set; }
}

22
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/BinaryVulnAssertionEntity.cs Normal file
View File

@@ -0,0 +1,22 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the binary_vuln_assertion table.
/// </summary>
public sealed class BinaryVulnAssertionEntity
{
public Guid Id { get; set; }
public Guid BinaryIdentityId { get; set; }
public string VulnId { get; set; } = null!;
public string Status { get; set; } = null!;
public string Source { get; set; } = null!;
public string AssertionType { get; set; } = null!;
public decimal Confidence { get; set; }
public string? EvidenceJson { get; set; }
public DateTimeOffset ValidFrom { get; set; }
public DateTimeOffset? ValidUntil { get; set; }
public string? SignatureRef { get; set; }
public DateTimeOffset CreatedAtUtc { get; set; }
public BinaryIdentityEntity? BinaryIdentity { get; set; }
}

19
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/CallGraphSnapshotEntity.cs Normal file
View File

@@ -0,0 +1,19 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the call_graph_snapshots table.
/// </summary>
public sealed class CallGraphSnapshotEntity
{
public Guid Id { get; set; }
public Guid TenantId { get; set; }
public string ScanId { get; set; } = null!;
public string Language { get; set; } = null!;
public string GraphDigest { get; set; } = null!;
public DateTimeOffset ExtractedAt { get; set; }
public int NodeCount { get; set; }
public int EdgeCount { get; set; }
public int EntrypointCount { get; set; }
public int SinkCount { get; set; }
public string SnapshotJson { get; set; } = null!;
}

17
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/IdempotencyKeyEntity.cs Normal file
View File

@@ -0,0 +1,17 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the scanner.idempotency_keys table.
/// </summary>
public sealed class IdempotencyKeyEntity
{
public Guid KeyId { get; set; }
public string TenantId { get; set; } = null!;
public string ContentDigest { get; set; } = null!;
public string EndpointPath { get; set; } = null!;
public int ResponseStatus { get; set; }
public string? ResponseBody { get; set; }
public string? ResponseHeaders { get; set; }
public DateTimeOffset CreatedAt { get; set; }
public DateTimeOffset ExpiresAt { get; set; }
}

24
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/MaterialRiskChangeEntity.cs Normal file
View File

@@ -0,0 +1,24 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the material_risk_changes table.
/// </summary>
public sealed class MaterialRiskChangeEntity
{
public Guid Id { get; set; }
public Guid TenantId { get; set; }
public string VulnId { get; set; } = null!;
public string Purl { get; set; } = null!;
public string ScanId { get; set; } = null!;
public bool HasMaterialChange { get; set; }
public decimal PriorityScore { get; set; }
public string PreviousStateHash { get; set; } = null!;
public string CurrentStateHash { get; set; } = null!;
public string Changes { get; set; } = null!;
public DateTimeOffset DetectedAt { get; set; }
public string? BaseScanId { get; set; }
public string? Cause { get; set; }
public string? CauseKind { get; set; }
public string? PathNodes { get; set; }
public string? AssociatedVulns { get; set; }
}

22
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/ProofBundleEntity.cs Normal file
View File

@@ -0,0 +1,22 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the proof_bundle table.
/// </summary>
public sealed class ProofBundleEntity
{
public Guid ScanId { get; set; }
public string RootHash { get; set; } = null!;
public string BundleType { get; set; } = null!;
public string? DsseEnvelope { get; set; }
public string? SignatureKeyId { get; set; }
public string? SignatureAlgorithm { get; set; }
public byte[]? BundleContent { get; set; }
public string BundleHash { get; set; } = null!;
public string? LedgerHash { get; set; }
public string? ManifestHash { get; set; }
public string? SbomHash { get; set; }
public string? VexHash { get; set; }
public DateTimeOffset CreatedAt { get; set; }
public DateTimeOffset? ExpiresAt { get; set; }
}

18
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/ReachabilityResultEntity.cs Normal file
View File

@@ -0,0 +1,18 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the reachability_results table.
/// </summary>
public sealed class ReachabilityResultEntity
{
public Guid Id { get; set; }
public Guid TenantId { get; set; }
public string ScanId { get; set; } = null!;
public string Language { get; set; } = null!;
public string GraphDigest { get; set; } = null!;
public string ResultDigest { get; set; } = null!;
public DateTimeOffset ComputedAt { get; set; }
public int ReachableNodeCount { get; set; }
public int ReachableSinkCount { get; set; }
public string ResultJson { get; set; } = null!;
}

24
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/RiskStateSnapshotEntity.cs Normal file
View File

@@ -0,0 +1,24 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the risk_state_snapshots table (both public and scanner schema).
/// </summary>
public sealed class RiskStateSnapshotEntity
{
public Guid Id { get; set; }
public Guid TenantId { get; set; }
public string VulnId { get; set; } = null!;
public string Purl { get; set; } = null!;
public string ScanId { get; set; } = null!;
public DateTimeOffset CapturedAt { get; set; }
public bool? Reachable { get; set; }
public string? LatticeState { get; set; }
public string VexStatus { get; set; } = null!;
public bool? InAffectedRange { get; set; }
public bool Kev { get; set; }
public decimal? EpssScore { get; set; }
public string[]? PolicyFlags { get; set; }
public string? PolicyDecision { get; set; }
public string StateHash { get; set; } = null!;
public DateTimeOffset CreatedAt { get; set; }
}

20
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/ScanManifestEntity.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the scan_manifest table.
/// </summary>
public sealed class ScanManifestEntity
{
public Guid ManifestId { get; set; }
public Guid ScanId { get; set; }
public string ManifestHash { get; set; } = null!;
public string SbomHash { get; set; } = null!;
public string RulesHash { get; set; } = null!;
public string FeedHash { get; set; } = null!;
public string PolicyHash { get; set; } = null!;
public DateTimeOffset ScanStartedAt { get; set; }
public DateTimeOffset? ScanCompletedAt { get; set; }
public string ManifestContent { get; set; } = null!;
public string ScannerVersion { get; set; } = null!;
public DateTimeOffset CreatedAt { get; set; }
}

36
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/ScanMetricsEntity.cs Normal file
View File

@@ -0,0 +1,36 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the scanner.scan_metrics table.
/// </summary>
public sealed class ScanMetricsEntity
{
public Guid MetricsId { get; set; }
public Guid ScanId { get; set; }
public Guid TenantId { get; set; }
public Guid? SurfaceId { get; set; }
public string ArtifactDigest { get; set; } = null!;
public string ArtifactType { get; set; } = null!;
public string? ReplayManifestHash { get; set; }
public string FindingsSha256 { get; set; } = null!;
public string? VexBundleSha256 { get; set; }
public string? ProofBundleSha256 { get; set; }
public string? SbomSha256 { get; set; }
public string? PolicyDigest { get; set; }
public string? FeedSnapshotId { get; set; }
public DateTimeOffset StartedAt { get; set; }
public DateTimeOffset FinishedAt { get; set; }
public int TIngestMs { get; set; }
public int TAnalyzeMs { get; set; }
public int TReachabilityMs { get; set; }
public int TVexMs { get; set; }
public int TSignMs { get; set; }
public int TPublishMs { get; set; }
public int? PackageCount { get; set; }
public int? FindingCount { get; set; }
public int? VexDecisionCount { get; set; }
public string ScannerVersion { get; set; } = null!;
public string? ScannerImageDigest { get; set; }
public bool IsReplay { get; set; }
public DateTimeOffset CreatedAt { get; set; }
}

24
src/Scanner/__Libraries/StellaOps.Scanner.Storage/EfCore/Models/SecretDetectionSettingsEntity.cs Normal file
View File

@@ -0,0 +1,24 @@
namespace StellaOps.Scanner.Storage.EfCore.Models;
/// <summary>
/// EF Core entity for the secret_detection_settings table.
/// </summary>
public sealed class SecretDetectionSettingsEntity
{
public Guid SettingsId { get; set; }
public Guid TenantId { get; set; }
public bool Enabled { get; set; }
public string? RevelationPolicy { get; set; }
public string[]? EnabledRuleCategories { get; set; }
public string[]? DisabledRuleIds { get; set; }
public string? AlertSettings { get; set; }
public long MaxFileSizeBytes { get; set; }
public string[]? ExcludedFileExtensions { get; set; }
public string[]? ExcludedPaths { get; set; }
public bool ScanBinaryFiles { get; set; }
public bool RequireSignedRuleBundles { get; set; }
public int Version { get; set; }
public DateTimeOffset UpdatedAt { get; set; }
public string? UpdatedBy { get; set; }
public DateTimeOffset CreatedAt { get; set; }
}

200
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/Migrations/022a_runtime_observations_compat.sql Normal file
View File

@@ -0,0 +1,200 @@
-- Compatibility bridge between 022_reachability_evidence and 023_runtime_observations.
-- 022 creates scanner.runtime_observations in the legacy shape; 023 expects node_hash/function_map columns.
CREATE SCHEMA IF NOT EXISTS scanner;
CREATE TABLE IF NOT EXISTS scanner.runtime_observations (
id UUID PRIMARY KEY DEFAULT gen_random_uuid()
);
ALTER TABLE scanner.runtime_observations
ADD COLUMN IF NOT EXISTS observation_id TEXT,
ADD COLUMN IF NOT EXISTS node_hash TEXT,
ADD COLUMN IF NOT EXISTS function_name TEXT,
ADD COLUMN IF NOT EXISTS pod_name TEXT,
ADD COLUMN IF NOT EXISTS namespace TEXT,
ADD COLUMN IF NOT EXISTS probe_type TEXT,
ADD COLUMN IF NOT EXISTS observation_count INTEGER DEFAULT 1,
ADD COLUMN IF NOT EXISTS duration_us BIGINT,
ADD COLUMN IF NOT EXISTS observed_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ DEFAULT now();
UPDATE scanner.runtime_observations
SET observation_id = COALESCE(observation_id, id::text)
WHERE observation_id IS NULL;
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'scanner'
AND table_name = 'runtime_observations'
AND column_name = 'symbol_name')
THEN
EXECUTE $sql$
UPDATE scanner.runtime_observations
SET function_name = COALESCE(function_name, symbol_name)
WHERE function_name IS NULL;
$sql$;
END IF;
END
$$;
UPDATE scanner.runtime_observations
SET function_name = COALESCE(function_name, 'unknown')
WHERE function_name IS NULL;
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'scanner'
AND table_name = 'runtime_observations'
AND column_name = 'observation_source')
THEN
EXECUTE $sql$
UPDATE scanner.runtime_observations
SET probe_type = COALESCE(probe_type, observation_source)
WHERE probe_type IS NULL;
$sql$;
END IF;
END
$$;
UPDATE scanner.runtime_observations
SET probe_type = COALESCE(probe_type, 'runtime')
WHERE probe_type IS NULL;
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'scanner'
AND table_name = 'runtime_observations'
AND column_name = 'last_observed_at_utc')
THEN
EXECUTE $sql$
UPDATE scanner.runtime_observations
SET observed_at = COALESCE(observed_at, last_observed_at_utc)
WHERE observed_at IS NULL;
$sql$;
END IF;
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'scanner'
AND table_name = 'runtime_observations'
AND column_name = 'first_observed_at_utc')
THEN
EXECUTE $sql$
UPDATE scanner.runtime_observations
SET observed_at = COALESCE(observed_at, first_observed_at_utc)
WHERE observed_at IS NULL;
$sql$;
END IF;
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'scanner'
AND table_name = 'runtime_observations'
AND column_name = 'created_at_utc')
THEN
EXECUTE $sql$
UPDATE scanner.runtime_observations
SET observed_at = COALESCE(observed_at, created_at_utc)
WHERE observed_at IS NULL;
$sql$;
EXECUTE $sql$
UPDATE scanner.runtime_observations
SET created_at = COALESCE(created_at, created_at_utc)
WHERE created_at IS NULL;
$sql$;
END IF;
END
$$;
UPDATE scanner.runtime_observations
SET observed_at = COALESCE(observed_at, now())
WHERE observed_at IS NULL;
UPDATE scanner.runtime_observations
SET created_at = COALESCE(created_at, now())
WHERE created_at IS NULL;
DO $$
BEGIN
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'scanner'
AND table_name = 'runtime_observations'
AND column_name = 'image_digest')
AND EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'scanner'
AND table_name = 'runtime_observations'
AND column_name = 'symbol_name')
THEN
EXECUTE $sql$
UPDATE scanner.runtime_observations
SET node_hash = COALESCE(
node_hash,
'legacy:' || md5(
COALESCE(image_digest, '') || '|' ||
COALESCE(symbol_name, '') || '|' ||
COALESCE(observation_id, '')))
WHERE node_hash IS NULL;
$sql$;
ELSIF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'scanner'
AND table_name = 'runtime_observations'
AND column_name = 'symbol_name')
THEN
EXECUTE $sql$
UPDATE scanner.runtime_observations
SET node_hash = COALESCE(
node_hash,
'legacy:' || md5(COALESCE(symbol_name, '') || '|' || COALESCE(observation_id, '')))
WHERE node_hash IS NULL;
$sql$;
END IF;
END
$$;
UPDATE scanner.runtime_observations
SET node_hash = COALESCE(node_hash, 'legacy:' || md5(COALESCE(observation_id, '')))
WHERE node_hash IS NULL;
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_indexes
WHERE schemaname = 'scanner'
AND tablename = 'runtime_observations'
AND indexdef ILIKE 'CREATE UNIQUE INDEX%'
AND indexdef ILIKE '%(observation_id)%')
THEN
EXECUTE 'CREATE UNIQUE INDEX uq_runtime_observations_observation_id ON scanner.runtime_observations (observation_id)';
END IF;
END
$$;
ALTER TABLE scanner.runtime_observations
ALTER COLUMN observation_id SET NOT NULL,
ALTER COLUMN node_hash SET NOT NULL,
ALTER COLUMN function_name SET NOT NULL,
ALTER COLUMN probe_type SET NOT NULL,
ALTER COLUMN observed_at SET NOT NULL,
ALTER COLUMN created_at SET NOT NULL,
ALTER COLUMN created_at SET DEFAULT now(),
ALTER COLUMN observation_count SET DEFAULT 1;

389
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresArtifactBomRepository.cs
View File

@@ -1,5 +1,6 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Scanner.Storage.Entities;
using StellaOps.Scanner.Storage.Repositories;
@@ -40,78 +41,55 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
var monthEnd = monthStart.AddMonths(1);
var lockKey = $"{row.CanonicalBomSha256}|{row.PayloadDigest}|{monthStart:yyyy-MM}";
const string selectExistingTemplate = """
var selectExistingSql = $"""
SELECT
build_id AS BuildId,
canonical_bom_sha256 AS CanonicalBomSha256,
payload_digest AS PayloadDigest,
inserted_at AS InsertedAt,
raw_bom_ref AS RawBomRef,
canonical_bom_ref AS CanonicalBomRef,
dsse_envelope_ref AS DsseEnvelopeRef,
merged_vex_ref AS MergedVexRef,
canonical_bom::text AS CanonicalBomJson,
merged_vex::text AS MergedVexJson,
attestations::text AS AttestationsJson,
evidence_score AS EvidenceScore,
rekor_tile_id AS RekorTileId
FROM {0}
WHERE canonical_bom_sha256 = @CanonicalBomSha256
AND payload_digest = @PayloadDigest
AND inserted_at >= @MonthStart
AND inserted_at < @MonthEnd
build_id AS "BuildId",
canonical_bom_sha256 AS "CanonicalBomSha256",
payload_digest AS "PayloadDigest",
inserted_at AS "InsertedAt",
raw_bom_ref AS "RawBomRef",
canonical_bom_ref AS "CanonicalBomRef",
dsse_envelope_ref AS "DsseEnvelopeRef",
merged_vex_ref AS "MergedVexRef",
canonical_bom::text AS "CanonicalBomJson",
merged_vex::text AS "MergedVexJson",
attestations::text AS "AttestationsJson",
evidence_score AS "EvidenceScore",
rekor_tile_id AS "RekorTileId"
FROM {TableName}
WHERE canonical_bom_sha256 = $1
AND payload_digest = $2
AND inserted_at >= $3
AND inserted_at < $4
ORDER BY inserted_at DESC, build_id ASC
LIMIT 1
FOR UPDATE
""";
var selectExistingSql = string.Format(selectExistingTemplate, TableName);
var updateExistingSql = $"""
UPDATE {TableName}
SET
raw_bom_ref = @RawBomRef,
canonical_bom_ref = @CanonicalBomRef,
dsse_envelope_ref = @DsseEnvelopeRef,
merged_vex_ref = @MergedVexRef,
canonical_bom = @CanonicalBomJson::jsonb,
merged_vex = @MergedVexJson::jsonb,
attestations = @AttestationsJson::jsonb,
evidence_score = @EvidenceScore,
rekor_tile_id = @RekorTileId
WHERE build_id = @BuildId
AND inserted_at = @InsertedAt
raw_bom_ref = $1,
canonical_bom_ref = $2,
dsse_envelope_ref = $3,
merged_vex_ref = $4,
canonical_bom = $5::jsonb,
merged_vex = $6::jsonb,
attestations = $7::jsonb,
evidence_score = $8,
rekor_tile_id = $9
WHERE build_id = $10
AND inserted_at = $11
""";
var insertSql = $"""
INSERT INTO {TableName} (
build_id,
canonical_bom_sha256,
payload_digest,
inserted_at,
raw_bom_ref,
canonical_bom_ref,
dsse_envelope_ref,
merged_vex_ref,
canonical_bom,
merged_vex,
attestations,
evidence_score,
rekor_tile_id
build_id, canonical_bom_sha256, payload_digest, inserted_at,
raw_bom_ref, canonical_bom_ref, dsse_envelope_ref, merged_vex_ref,
canonical_bom, merged_vex, attestations, evidence_score, rekor_tile_id
) VALUES (
@BuildId,
@CanonicalBomSha256,
@PayloadDigest,
@InsertedAt,
@RawBomRef,
@CanonicalBomRef,
@DsseEnvelopeRef,
@MergedVexRef,
@CanonicalBomJson::jsonb,
@MergedVexJson::jsonb,
@AttestationsJson::jsonb,
@EvidenceScore,
@RekorTileId
$1, $2, $3, $4, $5, $6, $7, $8,
$9::jsonb, $10::jsonb, $11::jsonb, $12, $13
)
ON CONFLICT (build_id, inserted_at) DO UPDATE SET
canonical_bom_sha256 = EXCLUDED.canonical_bom_sha256,
@@ -130,47 +108,59 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
var command = new CommandDefinition(
"SELECT pg_advisory_xact_lock(hashtext(@LockKey));",
new { LockKey = lockKey },
transaction,
cancellationToken: cancellationToken);
await connection.ExecuteAsync(command).ConfigureAwait(false);
// Advisory lock
await using (var lockCmd = new NpgsqlCommand("SELECT pg_advisory_xact_lock(hashtext($1));", connection, transaction))
{
lockCmd.Parameters.AddWithValue(lockKey);
await lockCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
var existing = await connection.QuerySingleOrDefaultAsync<ArtifactBomRow>(
new CommandDefinition(
selectExistingSql,
new
// Try to find existing row with FOR UPDATE
ArtifactBomRow? existing = null;
await using (var selectCmd = new NpgsqlCommand(selectExistingSql, connection, transaction))
{
selectCmd.Parameters.AddWithValue(row.CanonicalBomSha256);
selectCmd.Parameters.AddWithValue(row.PayloadDigest);
selectCmd.Parameters.AddWithValue(monthStart);
selectCmd.Parameters.AddWithValue(monthEnd);
await using var reader = await selectCmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
existing = new ArtifactBomRow
{
row.CanonicalBomSha256,
row.PayloadDigest,
MonthStart = monthStart,
MonthEnd = monthEnd
},
transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
BuildId = reader.GetString(0),
CanonicalBomSha256 = reader.GetString(1),
PayloadDigest = reader.GetString(2),
InsertedAt = reader.GetFieldValue<DateTimeOffset>(3),
RawBomRef = reader.IsDBNull(4) ? null : reader.GetString(4),
CanonicalBomRef = reader.IsDBNull(5) ? null : reader.GetString(5),
DsseEnvelopeRef = reader.IsDBNull(6) ? null : reader.GetString(6),
MergedVexRef = reader.IsDBNull(7) ? null : reader.GetString(7),
CanonicalBomJson = reader.IsDBNull(8) ? null : reader.GetString(8),
MergedVexJson = reader.IsDBNull(9) ? null : reader.GetString(9),
AttestationsJson = reader.IsDBNull(10) ? null : reader.GetString(10),
EvidenceScore = reader.IsDBNull(11) ? 0 : reader.GetInt32(11),
RekorTileId = reader.IsDBNull(12) ? null : reader.GetString(12)
};
}
}
if (existing is not null)
{
await connection.ExecuteAsync(
new CommandDefinition(
updateExistingSql,
new
{
BuildId = existing.BuildId,
InsertedAt = existing.InsertedAt,
row.RawBomRef,
row.CanonicalBomRef,
row.DsseEnvelopeRef,
row.MergedVexRef,
row.CanonicalBomJson,
row.MergedVexJson,
row.AttestationsJson,
row.EvidenceScore,
row.RekorTileId
},
transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var updateCmd = new NpgsqlCommand(updateExistingSql, connection, transaction);
updateCmd.Parameters.AddWithValue((object?)row.RawBomRef ?? DBNull.Value);
updateCmd.Parameters.AddWithValue((object?)row.CanonicalBomRef ?? DBNull.Value);
updateCmd.Parameters.AddWithValue((object?)row.DsseEnvelopeRef ?? DBNull.Value);
updateCmd.Parameters.AddWithValue((object?)row.MergedVexRef ?? DBNull.Value);
updateCmd.Parameters.AddWithValue((object?)row.CanonicalBomJson ?? DBNull.Value);
updateCmd.Parameters.AddWithValue((object?)row.MergedVexJson ?? DBNull.Value);
updateCmd.Parameters.AddWithValue((object?)row.AttestationsJson ?? DBNull.Value);
updateCmd.Parameters.AddWithValue(row.EvidenceScore);
updateCmd.Parameters.AddWithValue((object?)row.RekorTileId ?? DBNull.Value);
updateCmd.Parameters.AddWithValue(existing.BuildId);
updateCmd.Parameters.AddWithValue(existing.InsertedAt);
await updateCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
@@ -186,27 +176,23 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
return existing;
}
await connection.ExecuteAsync(
new CommandDefinition(
insertSql,
new
{
row.BuildId,
row.CanonicalBomSha256,
row.PayloadDigest,
InsertedAt = insertedAt,
row.RawBomRef,
row.CanonicalBomRef,
row.DsseEnvelopeRef,
row.MergedVexRef,
row.CanonicalBomJson,
row.MergedVexJson,
row.AttestationsJson,
row.EvidenceScore,
row.RekorTileId
},
transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using (var insertCmd = new NpgsqlCommand(insertSql, connection, transaction))
{
insertCmd.Parameters.AddWithValue(row.BuildId);
insertCmd.Parameters.AddWithValue(row.CanonicalBomSha256);
insertCmd.Parameters.AddWithValue(row.PayloadDigest);
insertCmd.Parameters.AddWithValue(insertedAt);
insertCmd.Parameters.AddWithValue((object?)row.RawBomRef ?? DBNull.Value);
insertCmd.Parameters.AddWithValue((object?)row.CanonicalBomRef ?? DBNull.Value);
insertCmd.Parameters.AddWithValue((object?)row.DsseEnvelopeRef ?? DBNull.Value);
insertCmd.Parameters.AddWithValue((object?)row.MergedVexRef ?? DBNull.Value);
insertCmd.Parameters.AddWithValue((object?)row.CanonicalBomJson ?? DBNull.Value);
insertCmd.Parameters.AddWithValue((object?)row.MergedVexJson ?? DBNull.Value);
insertCmd.Parameters.AddWithValue((object?)row.AttestationsJson ?? DBNull.Value);
insertCmd.Parameters.AddWithValue(row.EvidenceScore);
insertCmd.Parameters.AddWithValue((object?)row.RekorTileId ?? DBNull.Value);
await insertCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
@@ -222,24 +208,27 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
var sql = $"""
SELECT
build_id AS BuildId,
canonical_bom_sha256 AS CanonicalBomSha256,
payload_digest AS PayloadDigest,
inserted_at AS InsertedAt,
evidence_score AS EvidenceScore,
rekor_tile_id AS RekorTileId
build_id AS "BuildId",
canonical_bom_sha256 AS "CanonicalBomSha256",
payload_digest AS "PayloadDigest",
inserted_at AS "InsertedAt",
evidence_score AS "EvidenceScore",
rekor_tile_id AS "RekorTileId"
FROM {TableName}
WHERE payload_digest = @PayloadDigest
WHERE payload_digest = @p0
ORDER BY inserted_at DESC, build_id ASC
LIMIT 1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.QuerySingleOrDefaultAsync<ArtifactBomRow>(
new CommandDefinition(
sql,
new { PayloadDigest = payloadDigest.Trim() },
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var result = await dbContext.Database.SqlQueryRaw<ArtifactBomRow>(
sql, payloadDigest.Trim())
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return result?.BuildId is not null ? result : null;
}
public async Task<IReadOnlyList<ArtifactBomRow>> FindByComponentPurlAsync(
@@ -253,29 +242,30 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
var sql = $"""
SELECT
build_id AS BuildId,
canonical_bom_sha256 AS CanonicalBomSha256,
payload_digest AS PayloadDigest,
inserted_at AS InsertedAt,
evidence_score AS EvidenceScore
build_id AS "BuildId",
canonical_bom_sha256 AS "CanonicalBomSha256",
payload_digest AS "PayloadDigest",
inserted_at AS "InsertedAt",
evidence_score AS "EvidenceScore"
FROM {TableName}
WHERE jsonb_path_exists(
canonical_bom,
'$.components[*] ? (@.purl == $purl)',
jsonb_build_object('purl', to_jsonb(@Purl::text)))
jsonb_build_object('purl', to_jsonb(@p0::text)))
ORDER BY inserted_at DESC, build_id ASC
LIMIT @Limit
OFFSET @Offset
LIMIT @p1
OFFSET @p2
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<ArtifactBomRow>(
new CommandDefinition(
sql,
new { Purl = purl.Trim(), Limit = limit, Offset = offset },
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return rows.AsList();
var rows = await dbContext.Database.SqlQueryRaw<ArtifactBomRow>(
sql, purl.Trim(), limit, offset)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows;
}
public async Task<IReadOnlyList<ArtifactBomRow>> FindByComponentNameAsync(
@@ -295,38 +285,37 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
var sql = $"""
SELECT
build_id AS BuildId,
canonical_bom_sha256 AS CanonicalBomSha256,
payload_digest AS PayloadDigest,
inserted_at AS InsertedAt,
evidence_score AS EvidenceScore
build_id AS "BuildId",
canonical_bom_sha256 AS "CanonicalBomSha256",
payload_digest AS "PayloadDigest",
inserted_at AS "InsertedAt",
evidence_score AS "EvidenceScore"
FROM {TableName}
WHERE jsonb_path_exists(
canonical_bom,
@JsonPath::jsonpath,
@p0::jsonpath,
jsonb_build_object(
'name', to_jsonb(@Name::text),
'minVersion', to_jsonb(@MinVersion::text)))
'name', to_jsonb(@p1::text),
'minVersion', to_jsonb(@p2::text)))
ORDER BY inserted_at DESC, build_id ASC
LIMIT @Limit
OFFSET @Offset
LIMIT @p3
OFFSET @p4
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<ArtifactBomRow>(
new CommandDefinition(
sql,
new
{
JsonPath = jsonPath,
Name = componentName.Trim().ToLowerInvariant(),
MinVersion = minVersion?.Trim() ?? string.Empty,
Limit = limit,
Offset = offset
},
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return rows.AsList();
var rows = await dbContext.Database.SqlQueryRaw<ArtifactBomRow>(
sql,
jsonPath,
componentName.Trim().ToLowerInvariant(),
minVersion?.Trim() ?? string.Empty,
limit,
offset)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows;
}
public async Task<IReadOnlyList<ArtifactBomRow>> FindPendingTriageAsync(
@@ -340,27 +329,28 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
var sql = $"""
SELECT
build_id AS BuildId,
canonical_bom_sha256 AS CanonicalBomSha256,
payload_digest AS PayloadDigest,
inserted_at AS InsertedAt,
evidence_score AS EvidenceScore,
jsonb_path_query_array(merged_vex, @PendingPath::jsonpath)::text AS PendingMergedVexJson
build_id AS "BuildId",
canonical_bom_sha256 AS "CanonicalBomSha256",
payload_digest AS "PayloadDigest",
inserted_at AS "InsertedAt",
evidence_score AS "EvidenceScore",
jsonb_path_query_array(merged_vex, @p0::jsonpath)::text AS "PendingMergedVexJson"
FROM {TableName}
WHERE jsonb_path_exists(merged_vex, @PendingPath::jsonpath)
WHERE jsonb_path_exists(merged_vex, @p0::jsonpath)
ORDER BY inserted_at DESC, build_id ASC
LIMIT @Limit
OFFSET @Offset
LIMIT @p1
OFFSET @p2
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<ArtifactBomRow>(
new CommandDefinition(
sql,
new { PendingPath, Limit = limit, Offset = offset },
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return rows.AsList();
var rows = await dbContext.Database.SqlQueryRaw<ArtifactBomRow>(
sql, PendingPath, limit, offset)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows;
}
public async Task EnsureFuturePartitionsAsync(int monthsAhead, CancellationToken cancellationToken = default)
@@ -370,18 +360,24 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
throw new ArgumentOutOfRangeException(nameof(monthsAhead), "monthsAhead must be >= 0.");
}
var sql = $"SELECT partition_name FROM {SchemaName}.ensure_artifact_boms_future_partitions(@MonthsAhead);";
var sql = $"SELECT partition_name FROM {SchemaName}.ensure_artifact_boms_future_partitions($1);";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var partitions = await connection.QueryAsync<string>(
new CommandDefinition(
sql,
new { MonthsAhead = monthsAhead },
cancellationToken: cancellationToken)).ConfigureAwait(false);
var partitions = new List<string>();
await using (var cmd = new NpgsqlCommand(sql, connection))
{
cmd.Parameters.AddWithValue(monthsAhead);
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
partitions.Add(reader.GetString(0));
}
}
_logger.LogInformation(
"Ensured scanner.artifact_boms partitions monthsAhead={MonthsAhead} createdOrVerified={Count}",
monthsAhead,
partitions.Count());
partitions.Count);
}
public async Task<IReadOnlyList<ArtifactBomPartitionDropRow>> DropOldPartitionsAsync(
@@ -396,19 +392,20 @@ public sealed class PostgresArtifactBomRepository : IArtifactBomRepository
var sql = $"""
SELECT
partition_name AS PartitionName,
dropped AS Dropped
FROM {SchemaName}.drop_artifact_boms_partitions_older_than(@RetainMonths, @DryRun)
partition_name AS "PartitionName",
dropped AS "Dropped"
FROM {SchemaName}.drop_artifact_boms_partitions_older_than(@p0, @p1)
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<ArtifactBomPartitionDropRow>(
new CommandDefinition(
sql,
new { RetainMonths = retainMonths, DryRun = dryRun },
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return rows.AsList();
var rows = await dbContext.Database.SqlQueryRaw<ArtifactBomPartitionDropRow>(
sql, retainMonths, dryRun)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows;
}
private static void ValidatePagination(int limit, int offset)

436
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresBinaryEvidenceRepository.cs
View File

@@ -1,4 +1,5 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using StellaOps.Scanner.Storage.EfCore.Models;
using StellaOps.Scanner.Storage.Entities;
using StellaOps.Scanner.Storage.Repositories;
@@ -6,15 +7,12 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// PostgreSQL repository for binary evidence data.
/// Converted from Dapper to EF Core; INSERT RETURNING kept as raw SQL.
/// </summary>
public sealed class PostgresBinaryEvidenceRepository : IBinaryEvidenceRepository
{
private readonly ScannerDataSource _dataSource;
private string SchemaName => _dataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private string IdentityTable => $"{SchemaName}.binary_identity";
private string PackageMapTable => $"{SchemaName}.binary_package_map";
private string VulnAssertionTable => $"{SchemaName}.binary_vuln_assertion";
public PostgresBinaryEvidenceRepository(ScannerDataSource dataSource)
{
@@ -23,67 +21,87 @@ public sealed class PostgresBinaryEvidenceRepository : IBinaryEvidenceRepository
public async Task<BinaryIdentityRow?> GetByIdAsync(Guid id, CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
id AS Id,
scan_id AS ScanId,
file_path AS FilePath,
file_sha256 AS FileSha256,
text_sha256 AS TextSha256,
build_id AS BuildId,
build_id_type AS BuildIdType,
architecture AS Architecture,
binary_format AS BinaryFormat,
file_size AS FileSize,
is_stripped AS IsStripped,
has_debug_info AS HasDebugInfo,
created_at_utc AS CreatedAtUtc
FROM {IdentityTable}
WHERE id = @Id
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.QuerySingleOrDefaultAsync<BinaryIdentityRow>(
new CommandDefinition(sql, new { Id = id }, cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entity = await dbContext.BinaryIdentities
.AsNoTracking()
.FirstOrDefaultAsync(e => e.Id == id, cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapIdentityToRow(entity);
}
public Task<BinaryIdentityRow?> GetByBuildIdAsync(string buildId, CancellationToken cancellationToken = default)
=> GetByFieldAsync("build_id", buildId, cancellationToken);
public async Task<BinaryIdentityRow?> GetByBuildIdAsync(string buildId, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(buildId)) return null;
public Task<BinaryIdentityRow?> GetByFileSha256Async(string sha256, CancellationToken cancellationToken = default)
=> GetByFieldAsync("file_sha256", sha256, cancellationToken);
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
public Task<BinaryIdentityRow?> GetByTextSha256Async(string sha256, CancellationToken cancellationToken = default)
=> GetByFieldAsync("text_sha256", sha256, cancellationToken);
var entity = await dbContext.BinaryIdentities
.AsNoTracking()
.Where(e => e.BuildId == buildId)
.OrderByDescending(e => e.CreatedAtUtc)
.ThenBy(e => e.Id)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapIdentityToRow(entity);
}
public async Task<BinaryIdentityRow?> GetByFileSha256Async(string sha256, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(sha256)) return null;
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entity = await dbContext.BinaryIdentities
.AsNoTracking()
.Where(e => e.FileSha256 == sha256)
.OrderByDescending(e => e.CreatedAtUtc)
.ThenBy(e => e.Id)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapIdentityToRow(entity);
}
public async Task<BinaryIdentityRow?> GetByTextSha256Async(string sha256, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(sha256)) return null;
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entity = await dbContext.BinaryIdentities
.AsNoTracking()
.Where(e => e.TextSha256 == sha256)
.OrderByDescending(e => e.CreatedAtUtc)
.ThenBy(e => e.Id)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapIdentityToRow(entity);
}
public async Task<IReadOnlyList<BinaryIdentityRow>> GetByScanIdAsync(
Guid scanId,
CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
id AS Id,
scan_id AS ScanId,
file_path AS FilePath,
file_sha256 AS FileSha256,
text_sha256 AS TextSha256,
build_id AS BuildId,
build_id_type AS BuildIdType,
architecture AS Architecture,
binary_format AS BinaryFormat,
file_size AS FileSize,
is_stripped AS IsStripped,
has_debug_info AS HasDebugInfo,
created_at_utc AS CreatedAtUtc
FROM {IdentityTable}
WHERE scan_id = @ScanId
ORDER BY created_at_utc, id
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var results = await connection.QueryAsync<BinaryIdentityRow>(
new CommandDefinition(sql, new { ScanId = scanId }, cancellationToken: cancellationToken)).ConfigureAwait(false);
return results.ToList();
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entities = await dbContext.BinaryIdentities
.AsNoTracking()
.Where(e => e.ScanId == scanId)
.OrderBy(e => e.CreatedAtUtc)
.ThenBy(e => e.Id)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return entities.Select(MapIdentityToRow).ToList();
}
public async Task<BinaryIdentityRow> AddIdentityAsync(
@@ -92,41 +110,29 @@ public sealed class PostgresBinaryEvidenceRepository : IBinaryEvidenceRepository
{
ArgumentNullException.ThrowIfNull(identity);
var sql = $"""
INSERT INTO {IdentityTable} (
scan_id,
file_path,
file_sha256,
text_sha256,
build_id,
build_id_type,
architecture,
binary_format,
file_size,
is_stripped,
has_debug_info
) VALUES (
@ScanId,
@FilePath,
@FileSha256,
@TextSha256,
@BuildId,
@BuildIdType,
@Architecture,
@BinaryFormat,
@FileSize,
@IsStripped,
@HasDebugInfo
)
RETURNING id AS Id, created_at_utc AS CreatedAtUtc
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var created = await connection.QuerySingleAsync<(Guid Id, DateTimeOffset CreatedAtUtc)>(
new CommandDefinition(sql, identity, cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
identity.Id = created.Id;
identity.CreatedAtUtc = created.CreatedAtUtc;
var entity = new BinaryIdentityEntity
{
ScanId = identity.ScanId,
FilePath = identity.FilePath,
FileSha256 = identity.FileSha256,
TextSha256 = identity.TextSha256,
BuildId = identity.BuildId,
BuildIdType = identity.BuildIdType,
Architecture = identity.Architecture,
BinaryFormat = identity.BinaryFormat,
FileSize = identity.FileSize,
IsStripped = identity.IsStripped,
HasDebugInfo = identity.HasDebugInfo
};
dbContext.BinaryIdentities.Add(entity);
await dbContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
identity.Id = entity.Id;
identity.CreatedAtUtc = entity.CreatedAtUtc;
return identity;
}
@@ -134,26 +140,19 @@ public sealed class PostgresBinaryEvidenceRepository : IBinaryEvidenceRepository
Guid binaryId,
CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
id AS Id,
binary_identity_id AS BinaryIdentityId,
purl AS Purl,
match_type AS MatchType,
confidence AS Confidence,
match_source AS MatchSource,
evidence_json AS EvidenceJson,
created_at_utc AS CreatedAtUtc
FROM {PackageMapTable}
WHERE binary_identity_id = @BinaryIdentityId
ORDER BY created_at_utc, purl, id
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var results = await connection.QueryAsync<BinaryPackageMapRow>(
new CommandDefinition(sql, new { BinaryIdentityId = binaryId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entities = await dbContext.BinaryPackageMaps
.AsNoTracking()
.Where(e => e.BinaryIdentityId == binaryId)
.OrderBy(e => e.CreatedAtUtc)
.ThenBy(e => e.Purl)
.ThenBy(e => e.Id)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return results.ToList();
return entities.Select(MapPackageMapToRow).ToList();
}
public async Task<BinaryPackageMapRow> AddPackageMapAsync(
@@ -162,31 +161,27 @@ public sealed class PostgresBinaryEvidenceRepository : IBinaryEvidenceRepository
{
ArgumentNullException.ThrowIfNull(map);
// Keep raw SQL for jsonb cast in INSERT.
var sql = $"""
INSERT INTO {PackageMapTable} (
binary_identity_id,
purl,
match_type,
confidence,
match_source,
evidence_json
INSERT INTO binary_package_map (
binary_identity_id, purl, match_type, confidence, match_source, evidence_json
) VALUES (
@BinaryIdentityId,
@Purl,
@MatchType,
@Confidence,
@MatchSource,
@EvidenceJson::jsonb
@p0, @p1, @p2, @p3, @p4, @p5::jsonb
)
RETURNING id AS Id, created_at_utc AS CreatedAtUtc
RETURNING id, created_at_utc
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var created = await connection.QuerySingleAsync<(Guid Id, DateTimeOffset CreatedAtUtc)>(
new CommandDefinition(sql, map, cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
map.Id = created.Id;
map.CreatedAtUtc = created.CreatedAtUtc;
var result = await dbContext.Database.SqlQueryRaw<PackageMapInsertResult>(
sql, map.BinaryIdentityId, map.Purl, map.MatchType, map.Confidence,
map.MatchSource, (object?)map.EvidenceJson ?? DBNull.Value)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
map.Id = result.id;
map.CreatedAtUtc = result.created_at_utc;
return map;
}
@@ -194,60 +189,38 @@ public sealed class PostgresBinaryEvidenceRepository : IBinaryEvidenceRepository
Guid binaryId,
CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
id AS Id,
binary_identity_id AS BinaryIdentityId,
vuln_id AS VulnId,
status AS Status,
source AS Source,
assertion_type AS AssertionType,
confidence AS Confidence,
evidence_json AS EvidenceJson,
valid_from AS ValidFrom,
valid_until AS ValidUntil,
signature_ref AS SignatureRef,
created_at_utc AS CreatedAtUtc
FROM {VulnAssertionTable}
WHERE binary_identity_id = @BinaryIdentityId
ORDER BY created_at_utc, vuln_id, id
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var results = await connection.QueryAsync<BinaryVulnAssertionRow>(
new CommandDefinition(sql, new { BinaryIdentityId = binaryId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entities = await dbContext.BinaryVulnAssertions
.AsNoTracking()
.Where(e => e.BinaryIdentityId == binaryId)
.OrderBy(e => e.CreatedAtUtc)
.ThenBy(e => e.VulnId)
.ThenBy(e => e.Id)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return results.ToList();
return entities.Select(MapVulnAssertionToRow).ToList();
}
public async Task<IReadOnlyList<BinaryVulnAssertionRow>> GetVulnAssertionsByVulnIdAsync(
string vulnId,
CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
id AS Id,
binary_identity_id AS BinaryIdentityId,
vuln_id AS VulnId,
status AS Status,
source AS Source,
assertion_type AS AssertionType,
confidence AS Confidence,
evidence_json AS EvidenceJson,
valid_from AS ValidFrom,
valid_until AS ValidUntil,
signature_ref AS SignatureRef,
created_at_utc AS CreatedAtUtc
FROM {VulnAssertionTable}
WHERE vuln_id = @VulnId
ORDER BY created_at_utc, binary_identity_id, id
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var results = await connection.QueryAsync<BinaryVulnAssertionRow>(
new CommandDefinition(sql, new { VulnId = vulnId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entities = await dbContext.BinaryVulnAssertions
.AsNoTracking()
.Where(e => e.VulnId == vulnId)
.OrderBy(e => e.CreatedAtUtc)
.ThenBy(e => e.BinaryIdentityId)
.ThenBy(e => e.Id)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return results.ToList();
return entities.Select(MapVulnAssertionToRow).ToList();
}
public async Task<BinaryVulnAssertionRow> AddVulnAssertionAsync(
@@ -256,75 +229,88 @@ public sealed class PostgresBinaryEvidenceRepository : IBinaryEvidenceRepository
{
ArgumentNullException.ThrowIfNull(assertion);
// Keep raw SQL for jsonb cast in INSERT.
var sql = $"""
INSERT INTO {VulnAssertionTable} (
binary_identity_id,
vuln_id,
status,
source,
assertion_type,
confidence,
evidence_json,
valid_from,
valid_until,
signature_ref
INSERT INTO binary_vuln_assertion (
binary_identity_id, vuln_id, status, source, assertion_type,
confidence, evidence_json, valid_from, valid_until, signature_ref
) VALUES (
@BinaryIdentityId,
@VulnId,
@Status,
@Source,
@AssertionType,
@Confidence,
@EvidenceJson::jsonb,
@ValidFrom,
@ValidUntil,
@SignatureRef
@p0, @p1, @p2, @p3, @p4, @p5, @p6::jsonb, @p7, @p8, @p9
)
RETURNING id AS Id, created_at_utc AS CreatedAtUtc
RETURNING id, created_at_utc
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var created = await connection.QuerySingleAsync<(Guid Id, DateTimeOffset CreatedAtUtc)>(
new CommandDefinition(sql, assertion, cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
assertion.Id = created.Id;
assertion.CreatedAtUtc = created.CreatedAtUtc;
var result = await dbContext.Database.SqlQueryRaw<VulnAssertionInsertResult>(
sql, assertion.BinaryIdentityId, assertion.VulnId, assertion.Status,
assertion.Source, assertion.AssertionType, assertion.Confidence,
(object?)assertion.EvidenceJson ?? DBNull.Value,
assertion.ValidFrom, (object?)assertion.ValidUntil ?? DBNull.Value,
(object?)assertion.SignatureRef ?? DBNull.Value)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
assertion.Id = result.id;
assertion.CreatedAtUtc = result.created_at_utc;
return assertion;
}
private async Task<BinaryIdentityRow?> GetByFieldAsync(
string column,
string value,
CancellationToken cancellationToken)
private static BinaryIdentityRow MapIdentityToRow(BinaryIdentityEntity e) => new()
{
if (string.IsNullOrWhiteSpace(value))
{
return null;
}
Id = e.Id,
ScanId = e.ScanId,
FilePath = e.FilePath,
FileSha256 = e.FileSha256,
TextSha256 = e.TextSha256,
BuildId = e.BuildId,
BuildIdType = e.BuildIdType,
Architecture = e.Architecture,
BinaryFormat = e.BinaryFormat,
FileSize = e.FileSize,
IsStripped = e.IsStripped,
HasDebugInfo = e.HasDebugInfo,
CreatedAtUtc = e.CreatedAtUtc
};
var sql = $"""
SELECT
id AS Id,
scan_id AS ScanId,
file_path AS FilePath,
file_sha256 AS FileSha256,
text_sha256 AS TextSha256,
build_id AS BuildId,
build_id_type AS BuildIdType,
architecture AS Architecture,
binary_format AS BinaryFormat,
file_size AS FileSize,
is_stripped AS IsStripped,
has_debug_info AS HasDebugInfo,
created_at_utc AS CreatedAtUtc
FROM {IdentityTable}
WHERE {column} = @Value
ORDER BY created_at_utc DESC, id
LIMIT 1
""";
private static BinaryPackageMapRow MapPackageMapToRow(BinaryPackageMapEntity e) => new()
{
Id = e.Id,
BinaryIdentityId = e.BinaryIdentityId,
Purl = e.Purl,
MatchType = e.MatchType,
Confidence = e.Confidence,
MatchSource = e.MatchSource,
EvidenceJson = e.EvidenceJson,
CreatedAtUtc = e.CreatedAtUtc
};
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.QuerySingleOrDefaultAsync<BinaryIdentityRow>(
new CommandDefinition(sql, new { Value = value }, cancellationToken: cancellationToken)).ConfigureAwait(false);
private static BinaryVulnAssertionRow MapVulnAssertionToRow(BinaryVulnAssertionEntity e) => new()
{
Id = e.Id,
BinaryIdentityId = e.BinaryIdentityId,
VulnId = e.VulnId,
Status = e.Status,
Source = e.Source,
AssertionType = e.AssertionType,
Confidence = e.Confidence,
EvidenceJson = e.EvidenceJson,
ValidFrom = e.ValidFrom,
ValidUntil = e.ValidUntil,
SignatureRef = e.SignatureRef,
CreatedAtUtc = e.CreatedAtUtc
};
private sealed record PackageMapInsertResult
{
public Guid id { get; init; }
public DateTimeOffset created_at_utc { get; init; }
}
private sealed record VulnAssertionInsertResult
{
public Guid id { get; init; }
public DateTimeOffset created_at_utc { get; init; }
}
}

81
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresCallGraphSnapshotRepository.cs
View File

@@ -1,5 +1,5 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Contracts;
using StellaOps.Scanner.Storage.Repositories;
@@ -7,11 +7,11 @@ using System.Text.Json;
namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// Converted from Dapper to EF Core raw SQL; ON CONFLICT + jsonb cast kept as raw SQL.
/// </summary>
public sealed class PostgresCallGraphSnapshotRepository : ICallGraphSnapshotRepository
{
private const string TenantContext = "00000000-0000-0000-0000-000000000001";
private static readonly Guid TenantId = Guid.Parse(TenantContext);
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false
@@ -31,34 +31,18 @@ public sealed class PostgresCallGraphSnapshotRepository : ICallGraphSnapshotRepo
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task StoreAsync(CallGraphSnapshot snapshot, CancellationToken ct = default)
public async Task StoreAsync(CallGraphSnapshot snapshot, CancellationToken ct = default, string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(snapshot);
var trimmed = snapshot.Trimmed();
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
INSERT INTO {CallGraphSnapshotsTable} (
tenant_id,
scan_id,
language,
graph_digest,
extracted_at,
node_count,
edge_count,
entrypoint_count,
sink_count,
snapshot_json
tenant_id, scan_id, language, graph_digest, extracted_at,
node_count, edge_count, entrypoint_count, sink_count, snapshot_json
) VALUES (
@TenantId,
@ScanId,
@Language,
@GraphDigest,
@ExtractedAt,
@NodeCount,
@EdgeCount,
@EntrypointCount,
@SinkCount,
@SnapshotJson::jsonb
@p0, @p1, @p2, @p3, @p4, @p5, @p6, @p7, @p8, @p9::jsonb
)
ON CONFLICT (tenant_id, scan_id, language, graph_digest) DO UPDATE SET
extracted_at = EXCLUDED.extracted_at,
@@ -71,20 +55,19 @@ public sealed class PostgresCallGraphSnapshotRepository : ICallGraphSnapshotRepo
var json = JsonSerializer.Serialize(trimmed, JsonOptions);
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
await connection.ExecuteAsync(new CommandDefinition(sql, new
{
TenantId = TenantId,
ScanId = trimmed.ScanId,
Language = trimmed.Language,
GraphDigest = trimmed.GraphDigest,
ExtractedAt = trimmed.ExtractedAt.UtcDateTime,
NodeCount = trimmed.Nodes.Length,
EdgeCount = trimmed.Edges.Length,
EntrypointCount = trimmed.EntrypointIds.Length,
SinkCount = trimmed.SinkIds.Length,
SnapshotJson = json
}, cancellationToken: ct)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
await dbContext.Database.ExecuteSqlRawAsync(
sql,
[
tenantScope.TenantId, trimmed.ScanId, trimmed.Language,
trimmed.GraphDigest, trimmed.ExtractedAt.UtcDateTime,
trimmed.Nodes.Length, trimmed.Edges.Length,
trimmed.EntrypointIds.Length, trimmed.SinkIds.Length,
json
],
ct).ConfigureAwait(false);
_logger.LogDebug(
"Stored call graph snapshot scan={ScanId} lang={Language} nodes={Nodes} edges={Edges}",
@@ -94,26 +77,27 @@ public sealed class PostgresCallGraphSnapshotRepository : ICallGraphSnapshotRepo
trimmed.Edges.Length);
}
public async Task<CallGraphSnapshot?> TryGetLatestAsync(string scanId, string language, CancellationToken ct = default)
public async Task<CallGraphSnapshot?> TryGetLatestAsync(string scanId, string language, CancellationToken ct = default, string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
ArgumentException.ThrowIfNullOrWhiteSpace(language);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT snapshot_json
FROM {CallGraphSnapshotsTable}
WHERE tenant_id = @TenantId AND scan_id = @ScanId AND language = @Language
WHERE tenant_id = @p0 AND scan_id = @p1 AND language = @p2
ORDER BY extracted_at DESC
LIMIT 1
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var json = await connection.ExecuteScalarAsync<string?>(new CommandDefinition(sql, new
{
TenantId = TenantId,
ScanId = scanId,
Language = language
}, cancellationToken: ct)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var json = await dbContext.Database.SqlQueryRaw<string>(
sql, tenantScope.TenantId, scanId, language)
.FirstOrDefaultAsync(ct)
.ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(json))
{
@@ -123,4 +107,3 @@ public sealed class PostgresCallGraphSnapshotRepository : ICallGraphSnapshotRepo
return JsonSerializer.Deserialize<CallGraphSnapshot>(json, JsonOptions);
}
}

67
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresCodeChangeRepository.cs
View File

@@ -1,6 +1,7 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Scanner.ReachabilityDrift;
using StellaOps.Scanner.Storage.Repositories;
using System.Text.Json;
@@ -9,9 +10,6 @@ namespace StellaOps.Scanner.Storage.Postgres;
public sealed class PostgresCodeChangeRepository : ICodeChangeRepository
{
private const string TenantContext = "00000000-0000-0000-0000-000000000001";
private static readonly Guid TenantId = Guid.Parse(TenantContext);
private readonly ScannerDataSource _dataSource;
private readonly ILogger<PostgresCodeChangeRepository> _logger;
@@ -26,7 +24,7 @@ public sealed class PostgresCodeChangeRepository : ICodeChangeRepository
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task StoreAsync(IReadOnlyList<CodeChangeFact> changes, CancellationToken ct = default)
public async Task StoreAsync(IReadOnlyList<CodeChangeFact> changes, CancellationToken ct = default, string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(changes);
@@ -34,32 +32,14 @@ public sealed class PostgresCodeChangeRepository : ICodeChangeRepository
{
return;
}
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
INSERT INTO {CodeChangesTable} (
id,
tenant_id,
scan_id,
base_scan_id,
language,
node_id,
file,
symbol,
change_kind,
details,
detected_at
id, tenant_id, scan_id, base_scan_id, language,
node_id, file, symbol, change_kind, details, detected_at
) VALUES (
@Id,
@TenantId,
@ScanId,
@BaseScanId,
@Language,
@NodeId,
@File,
@Symbol,
@ChangeKind,
@Details::jsonb,
@DetectedAt
@p0, @p1, @p2, @p3, @p4, @p5, @p6, @p7, @p8, @p9::jsonb, @p10
)
ON CONFLICT (tenant_id, scan_id, base_scan_id, language, symbol, change_kind) DO UPDATE SET
node_id = EXCLUDED.node_id,
@@ -68,23 +48,24 @@ public sealed class PostgresCodeChangeRepository : ICodeChangeRepository
detected_at = EXCLUDED.detected_at
""";
var rows = changes.Select(change => new
{
change.Id,
TenantId,
ScanId = change.ScanId.Trim(),
BaseScanId = change.BaseScanId.Trim(),
Language = change.Language.Trim(),
NodeId = string.IsNullOrWhiteSpace(change.NodeId) ? null : change.NodeId.Trim(),
File = change.File.Trim(),
Symbol = change.Symbol.Trim(),
ChangeKind = ToDbValue(change.Kind),
Details = SerializeDetails(change.Details),
DetectedAt = change.DetectedAt.UtcDateTime
}).ToList();
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
await connection.ExecuteAsync(new CommandDefinition(sql, rows, cancellationToken: ct)).ConfigureAwait(false);
foreach (var change in changes)
{
await dbContext.Database.ExecuteSqlRawAsync(
sql,
[
change.Id, tenantScope.TenantId, change.ScanId.Trim(),
change.BaseScanId.Trim(), change.Language.Trim(),
(object?)(string.IsNullOrWhiteSpace(change.NodeId) ? null : change.NodeId.Trim()) ?? DBNull.Value,
change.File.Trim(), change.Symbol.Trim(),
ToDbValue(change.Kind),
(object?)SerializeDetails(change.Details) ?? DBNull.Value,
change.DetectedAt.UtcDateTime
],
ct).ConfigureAwait(false);
}
_logger.LogDebug(
"Stored {Count} code change facts scan={ScanId} base={BaseScanId} lang={Language}",

147
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRawRepository.cs
View File

@@ -5,7 +5,8 @@
// Description: PostgreSQL implementation of IEpssRawRepository.
// -----------------------------------------------------------------------------
using Dapper;
using Microsoft.EntityFrameworkCore;
using Npgsql;
using StellaOps.Scanner.Storage.Repositories;
namespace StellaOps.Scanner.Storage.Postgres;
@@ -36,9 +37,9 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository
row_count, compressed_size, decompressed_size, import_run_id
)
VALUES (
@SourceUri, @AsOfDate, @Payload::jsonb, @PayloadSha256,
@HeaderComment, @ModelVersion, @PublishedDate,
@RowCount, @CompressedSize, @DecompressedSize, @ImportRunId
$1, $2, $3::jsonb, $4,
$5, $6, $7,
$8, $9, $10, $11
)
ON CONFLICT (source_uri, asof_date, payload_sha256) DO NOTHING
RETURNING raw_id, ingestion_ts
@@ -46,27 +47,28 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var result = await connection.QueryFirstOrDefaultAsync<(long raw_id, DateTimeOffset ingestion_ts)?>(sql, new
{
raw.SourceUri,
AsOfDate = raw.AsOfDate.ToDateTime(TimeOnly.MinValue),
raw.Payload,
raw.PayloadSha256,
raw.HeaderComment,
raw.ModelVersion,
PublishedDate = raw.PublishedDate?.ToDateTime(TimeOnly.MinValue),
raw.RowCount,
raw.CompressedSize,
raw.DecompressedSize,
raw.ImportRunId
});
await using var cmd = new NpgsqlCommand(sql, connection);
cmd.Parameters.AddWithValue(raw.SourceUri);
cmd.Parameters.AddWithValue(raw.AsOfDate.ToDateTime(TimeOnly.MinValue));
cmd.Parameters.AddWithValue(raw.Payload);
cmd.Parameters.AddWithValue(raw.PayloadSha256);
cmd.Parameters.AddWithValue((object?)raw.HeaderComment ?? DBNull.Value);
cmd.Parameters.AddWithValue((object?)raw.ModelVersion ?? DBNull.Value);
cmd.Parameters.AddWithValue(raw.PublishedDate.HasValue ? raw.PublishedDate.Value.ToDateTime(TimeOnly.MinValue) : DBNull.Value);
cmd.Parameters.AddWithValue(raw.RowCount);
cmd.Parameters.AddWithValue(raw.CompressedSize.HasValue ? raw.CompressedSize.Value : DBNull.Value);
cmd.Parameters.AddWithValue(raw.DecompressedSize.HasValue ? raw.DecompressedSize.Value : DBNull.Value);
cmd.Parameters.AddWithValue(raw.ImportRunId.HasValue ? raw.ImportRunId.Value : DBNull.Value);
if (result.HasValue)
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var rawId = reader.GetInt64(0);
var ingestionTs = reader.GetFieldValue<DateTimeOffset>(1);
return raw with
{
RawId = result.Value.raw_id,
IngestionTs = result.Value.ingestion_ts
RawId = rawId,
IngestionTs = ingestionTs
};
}
@@ -83,18 +85,20 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository
payload, payload_sha256, header_comment, model_version, published_date,
row_count, compressed_size, decompressed_size, import_run_id
FROM {RawTable}
WHERE asof_date = @AsOfDate
WHERE asof_date = @p0
ORDER BY ingestion_ts DESC
LIMIT 1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var row = await connection.QueryFirstOrDefaultAsync<RawRow?>(sql, new
{
AsOfDate = asOfDate.ToDateTime(TimeOnly.MinValue)
});
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return row.HasValue ? MapToRaw(row.Value) : null;
var row = await dbContext.Database.SqlQueryRaw<RawRow>(
sql, asOfDate.ToDateTime(TimeOnly.MinValue))
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return row is not null && row.raw_id != 0 ? MapToRaw(row) : null;
}
public async Task<IReadOnlyList<EpssRaw>> GetByDateRangeAsync(
@@ -108,16 +112,17 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository
payload, payload_sha256, header_comment, model_version, published_date,
row_count, compressed_size, decompressed_size, import_run_id
FROM {RawTable}
WHERE asof_date >= @StartDate AND asof_date <= @EndDate
WHERE asof_date >= @p0 AND asof_date <= @p1
ORDER BY asof_date DESC
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var rows = await connection.QueryAsync<RawRow>(sql, new
{
StartDate = startDate.ToDateTime(TimeOnly.MinValue),
EndDate = endDate.ToDateTime(TimeOnly.MinValue)
});
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<RawRow>(
sql, startDate.ToDateTime(TimeOnly.MinValue), endDate.ToDateTime(TimeOnly.MinValue))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows.Select(MapToRaw).ToList();
}
@@ -135,26 +140,30 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var row = await connection.QueryFirstOrDefaultAsync<RawRow?>(sql);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return row.HasValue ? MapToRaw(row.Value) : null;
var row = await dbContext.Database.SqlQueryRaw<RawRow>(sql)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return row is not null && row.raw_id != 0 ? MapToRaw(row) : null;
}
public async Task<bool> ExistsAsync(DateOnly asOfDate, byte[] payloadSha256, CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT EXISTS (
SELECT CAST(CASE WHEN EXISTS (
SELECT 1 FROM {RawTable}
WHERE asof_date = @AsOfDate AND payload_sha256 = @PayloadSha256
)
WHERE asof_date = $1 AND payload_sha256 = $2
) THEN 1 ELSE 0 END AS integer)
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
return await connection.ExecuteScalarAsync<bool>(sql, new
{
AsOfDate = asOfDate.ToDateTime(TimeOnly.MinValue),
PayloadSha256 = payloadSha256
});
await using var cmd = new NpgsqlCommand(sql, connection);
cmd.Parameters.AddWithValue(asOfDate.ToDateTime(TimeOnly.MinValue));
cmd.Parameters.AddWithValue(payloadSha256);
var result = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return Convert.ToInt32(result) == 1;
}
public async Task<IReadOnlyList<EpssRaw>> GetByModelVersionAsync(
@@ -168,27 +177,31 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository
payload, payload_sha256, header_comment, model_version, published_date,
row_count, compressed_size, decompressed_size, import_run_id
FROM {RawTable}
WHERE model_version = @ModelVersion
WHERE model_version = @p0
ORDER BY asof_date DESC
LIMIT @Limit
LIMIT @p1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var rows = await connection.QueryAsync<RawRow>(sql, new
{
ModelVersion = modelVersion,
Limit = limit
});
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<RawRow>(
sql, modelVersion, limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows.Select(MapToRaw).ToList();
}
public async Task<int> PruneAsync(int retentionDays = 365, CancellationToken cancellationToken = default)
{
var sql = $"SELECT {SchemaName}.prune_epss_raw(@RetentionDays)";
var sql = $"SELECT {SchemaName}.prune_epss_raw($1)";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
return await connection.ExecuteScalarAsync<int>(sql, new { RetentionDays = retentionDays });
await using var cmd = new NpgsqlCommand(sql, connection);
cmd.Parameters.AddWithValue(retentionDays);
var result = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return Convert.ToInt32(result);
}
private static EpssRaw MapToRaw(RawRow row)
@@ -211,18 +224,20 @@ public sealed class PostgresEpssRawRepository : IEpssRawRepository
};
}
private readonly record struct RawRow(
long raw_id,
string source_uri,
DateTime asof_date,
DateTimeOffset ingestion_ts,
string payload,
byte[] payload_sha256,
string? header_comment,
string? model_version,
DateTime? published_date,
int row_count,
long? compressed_size,
long? decompressed_size,
Guid? import_run_id);
private sealed class RawRow
{
public long raw_id { get; set; }
public string source_uri { get; set; } = "";
public DateTime asof_date { get; set; }
public DateTimeOffset ingestion_ts { get; set; }
public string payload { get; set; } = "";
public byte[] payload_sha256 { get; set; } = [];
public string? header_comment { get; set; }
public string? model_version { get; set; }
public DateTime? published_date { get; set; }
public int row_count { get; set; }
public long? compressed_size { get; set; }
public long? decompressed_size { get; set; }
public Guid? import_run_id { get; set; }
}
}

290
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssRepository.cs
View File

@@ -6,20 +6,17 @@
// -----------------------------------------------------------------------------
using Dapper;
using Microsoft.EntityFrameworkCore;
using Npgsql;
using NpgsqlTypes;
using StellaOps.Scanner.Core.Epss;
using StellaOps.Scanner.Storage.Epss;
using StellaOps.Scanner.Storage.Repositories;
using System.Data;
namespace StellaOps.Scanner.Storage.Postgres;
public sealed class PostgresEpssRepository : IEpssRepository
{
private static int _typeHandlersRegistered;
private readonly ScannerDataSource _dataSource;
private string SchemaName => _dataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
@@ -31,7 +28,6 @@ public sealed class PostgresEpssRepository : IEpssRepository
public PostgresEpssRepository(ScannerDataSource dataSource)
{
EnsureTypeHandlers();
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
}
@@ -52,12 +48,16 @@ public sealed class PostgresEpssRepository : IEpssRepository
error,
created_at
FROM {ImportRunsTable}
WHERE model_date = @ModelDate
WHERE model_date = @p0
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var row = await connection.QuerySingleOrDefaultAsync<ImportRunRow>(
new CommandDefinition(sql, new { ModelDate = modelDate }, cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var row = await dbContext.Database.SqlQueryRaw<ImportRunRow>(
sql, modelDate)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return row?.ToModel();
}
@@ -81,13 +81,7 @@ public sealed class PostgresEpssRepository : IEpssRepository
status,
created_at
) VALUES (
@ModelDate,
@SourceUri,
@RetrievedAtUtc,
@FileSha256,
0,
'PENDING',
@RetrievedAtUtc
@p0, @p1, @p2, @p3, 0, 'PENDING', @p2
)
ON CONFLICT (model_date) DO UPDATE SET
source_uri = EXCLUDED.source_uri,
@@ -116,16 +110,12 @@ public sealed class PostgresEpssRepository : IEpssRepository
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var row = await connection.QuerySingleOrDefaultAsync<ImportRunRow>(new CommandDefinition(
insertSql,
new
{
ModelDate = modelDate,
SourceUri = sourceUri,
RetrievedAtUtc = retrievedAtUtc,
FileSha256 = fileSha256
},
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var row = await dbContext.Database.SqlQueryRaw<ImportRunRow>(
insertSql, modelDate, sourceUri, retrievedAtUtc, fileSha256)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (row is not null)
{
@@ -154,25 +144,26 @@ public sealed class PostgresEpssRepository : IEpssRepository
UPDATE {ImportRunsTable}
SET status = 'SUCCEEDED',
error = NULL,
row_count = @RowCount,
decompressed_sha256 = @DecompressedSha256,
model_version_tag = @ModelVersionTag,
published_date = @PublishedDate
WHERE import_run_id = @ImportRunId
row_count = @p0,
decompressed_sha256 = @p1,
model_version_tag = @p2,
published_date = @p3
WHERE import_run_id = @p4
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await connection.ExecuteAsync(new CommandDefinition(
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
await dbContext.Database.ExecuteSqlRawAsync(
sql,
new
{
ImportRunId = importRunId,
RowCount = rowCount,
DecompressedSha256 = decompressedSha256,
ModelVersionTag = modelVersionTag,
PublishedDate = publishedDate
},
cancellationToken: cancellationToken)).ConfigureAwait(false);
[
rowCount,
(object?)decompressedSha256 ?? DBNull.Value,
(object?)modelVersionTag ?? DBNull.Value,
publishedDate.HasValue ? publishedDate.Value : DBNull.Value,
importRunId
],
cancellationToken).ConfigureAwait(false);
}
public async Task MarkImportFailedAsync(Guid importRunId, string error, CancellationToken cancellationToken = default)
@@ -182,15 +173,15 @@ public sealed class PostgresEpssRepository : IEpssRepository
var sql = $"""
UPDATE {ImportRunsTable}
SET status = 'FAILED',
error = @Error
WHERE import_run_id = @ImportRunId
error = @p0
WHERE import_run_id = @p1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await connection.ExecuteAsync(new CommandDefinition(
sql,
new { ImportRunId = importRunId, Error = error },
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
await dbContext.Database.ExecuteSqlRawAsync(
sql, [error, importRunId], cancellationToken).ConfigureAwait(false);
}
public async Task<EpssWriteResult> WriteSnapshotAsync(
@@ -218,10 +209,10 @@ public sealed class PostgresEpssRepository : IEpssRepository
) ON COMMIT DROP
""";
await connection.ExecuteAsync(new CommandDefinition(
createStageSql,
transaction: transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using (var createCmd = new NpgsqlCommand(createStageSql, connection, transaction))
{
await createCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
var (rowCount, distinctCount) = await CopyStageAsync(connection, transaction, stageTable, rows, cancellationToken).ConfigureAwait(false);
if (rowCount != distinctCount)
@@ -231,15 +222,16 @@ public sealed class PostgresEpssRepository : IEpssRepository
var insertScoresSql = $"""
INSERT INTO {ScoresTable} (model_date, cve_id, epss_score, percentile, import_run_id)
SELECT @ModelDate, cve_id, epss_score, percentile, @ImportRunId
SELECT $1, cve_id, epss_score, percentile, $2
FROM {stageTable}
""";
await connection.ExecuteAsync(new CommandDefinition(
insertScoresSql,
new { ModelDate = modelDate, ImportRunId = importRunId },
transaction: transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using (var insertScoresCmd = new NpgsqlCommand(insertScoresSql, connection, transaction))
{
insertScoresCmd.Parameters.AddWithValue(NpgsqlDbType.Date, modelDate);
insertScoresCmd.Parameters.AddWithValue(importRunId);
await insertScoresCmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
await InsertChangesAsync(connection, transaction, stageTable, modelDate, importRunId, cancellationToken).ConfigureAwait(false);
await UpsertCurrentAsync(connection, transaction, stageTable, modelDate, importRunId, updatedAtUtc, cancellationToken).ConfigureAwait(false);
@@ -279,15 +271,17 @@ public sealed class PostgresEpssRepository : IEpssRepository
var sql = $"""
SELECT cve_id, epss_score, percentile, model_date, import_run_id
FROM {CurrentTable}
WHERE cve_id = ANY(@CveIds)
WHERE cve_id = ANY(@p0)
ORDER BY cve_id
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<CurrentRow>(new CommandDefinition(
sql,
new { CveIds = normalized },
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<CurrentRow>(
sql, (object)normalized)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var result = new Dictionary<string, EpssCurrentEntry>(StringComparer.Ordinal);
foreach (var row in rows)
@@ -316,16 +310,18 @@ public sealed class PostgresEpssRepository : IEpssRepository
var sql = $"""
SELECT model_date, epss_score, percentile, import_run_id
FROM {ScoresTable}
WHERE cve_id = @CveId
WHERE cve_id = @p0
ORDER BY model_date DESC
LIMIT @Limit
LIMIT @p1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rows = await connection.QueryAsync<HistoryRow>(new CommandDefinition(
sql,
new { CveId = normalized, Limit = limit },
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<HistoryRow>(
sql, normalized, limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows.Select(static row => new EpssHistoryEntry(
row.model_date,
@@ -341,12 +337,11 @@ public sealed class PostgresEpssRepository : IEpssRepository
DateOnly modelDate,
CancellationToken cancellationToken)
{
var sql = "SELECT create_epss_partition(@Year, @Month)";
await connection.ExecuteAsync(new CommandDefinition(
sql,
new { Year = modelDate.Year, Month = modelDate.Month },
transaction: transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
var sql = "SELECT create_epss_partition($1, $2)";
await using var cmd = new NpgsqlCommand(sql, connection, transaction);
cmd.Parameters.AddWithValue(modelDate.Year);
cmd.Parameters.AddWithValue(modelDate.Month);
await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private static async Task<(int RowCount, int DistinctCount)> CopyStageAsync(
@@ -372,17 +367,12 @@ public sealed class PostgresEpssRepository : IEpssRepository
await importer.CompleteAsync(cancellationToken).ConfigureAwait(false);
}
var countsSql = $"""
SELECT COUNT(*) AS total, COUNT(DISTINCT cve_id) AS distinct_count
FROM {stageTable}
""";
var countsSql = $"SELECT COUNT(DISTINCT cve_id) FROM {stageTable}";
await using var countCmd = new NpgsqlCommand(countsSql, connection, transaction);
var distinctObj = await countCmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
var distinctCount = Convert.ToInt32(distinctObj);
var counts = await connection.QuerySingleAsync<StageCounts>(new CommandDefinition(
countsSql,
transaction: transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
return (rowCount, counts.distinct_count);
return (rowCount, distinctCount);
}
private async Task InsertChangesAsync(
@@ -407,7 +397,7 @@ public sealed class PostgresEpssRepository : IEpssRepository
import_run_id
)
SELECT
@ModelDate,
$1,
s.cve_id,
c.epss_score AS old_score,
s.epss_score AS new_score,
@@ -424,7 +414,7 @@ public sealed class PostgresEpssRepository : IEpssRepository
cfg.high_percentile,
cfg.big_jump_delta
) AS flags,
@ImportRunId
$2
FROM {stageTable} s
LEFT JOIN {CurrentTable} c ON c.cve_id = s.cve_id
CROSS JOIN (
@@ -435,11 +425,10 @@ public sealed class PostgresEpssRepository : IEpssRepository
) cfg
""";
await connection.ExecuteAsync(new CommandDefinition(
sql,
new { ModelDate = modelDate, ImportRunId = importRunId },
transaction: transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var cmd = new NpgsqlCommand(sql, connection, transaction);
cmd.Parameters.AddWithValue(NpgsqlDbType.Date, modelDate);
cmd.Parameters.AddWithValue(importRunId);
await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
private async Task UpsertCurrentAsync(
@@ -464,9 +453,9 @@ public sealed class PostgresEpssRepository : IEpssRepository
cve_id,
epss_score,
percentile,
@ModelDate,
@ImportRunId,
@UpdatedAtUtc
$1,
$2,
$3
FROM {stageTable}
ON CONFLICT (cve_id) DO UPDATE SET
epss_score = EXCLUDED.epss_score,
@@ -476,11 +465,11 @@ public sealed class PostgresEpssRepository : IEpssRepository
updated_at = EXCLUDED.updated_at
""";
await connection.ExecuteAsync(new CommandDefinition(
sql,
new { ModelDate = modelDate, ImportRunId = importRunId, UpdatedAtUtc = updatedAtUtc },
transaction: transaction,
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var cmd = new NpgsqlCommand(sql, connection, transaction);
cmd.Parameters.AddWithValue(NpgsqlDbType.Date, modelDate);
cmd.Parameters.AddWithValue(importRunId);
cmd.Parameters.AddWithValue(updatedAtUtc);
await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
@@ -490,6 +479,17 @@ public sealed class PostgresEpssRepository : IEpssRepository
int limit = 100000,
CancellationToken cancellationToken = default)
{
var paramList = new List<object> { modelDate };
var paramIndex = 1;
var flagsClause = "";
if (flags.HasValue)
{
flagsClause = $"AND (flags & @p{paramIndex}) != 0";
paramList.Add((int)flags.Value);
paramIndex++;
}
var sql = $"""
SELECT
cve_id,
@@ -500,23 +500,21 @@ public sealed class PostgresEpssRepository : IEpssRepository
new_percentile,
model_date
FROM {ChangesTable}
WHERE model_date = @ModelDate
{(flags.HasValue ? "AND (flags & @Flags) != 0" : "")}
WHERE model_date = @p0
{flagsClause}
ORDER BY new_score DESC, cve_id
LIMIT @Limit
LIMIT @p{paramIndex}
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
paramList.Add(limit);
var rows = await connection.QueryAsync<ChangeRow>(new CommandDefinition(
sql,
new
{
ModelDate = modelDate,
Flags = flags.HasValue ? (int)flags.Value : 0,
Limit = limit
},
cancellationToken: cancellationToken)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<ChangeRow>(
sql, paramList.ToArray())
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows.Select(r => new EpssChangeRecord
{
@@ -569,11 +567,6 @@ public sealed class PostgresEpssRepository : IEpssRepository
return Core.Epss.EpssPriorityBand.Low;
}
private sealed class StageCounts
{
public int distinct_count { get; set; }
}
private sealed class ImportRunRow
{
public Guid import_run_id { get; set; }
@@ -621,69 +614,4 @@ public sealed class PostgresEpssRepository : IEpssRepository
public Guid import_run_id { get; set; }
}
private static void EnsureTypeHandlers()
{
if (Interlocked.Exchange(ref _typeHandlersRegistered, 1) == 1)
{
return;
}
SqlMapper.AddTypeHandler(new DateOnlyTypeHandler());
SqlMapper.AddTypeHandler(new NullableDateOnlyTypeHandler());
}
private sealed class DateOnlyTypeHandler : SqlMapper.TypeHandler<DateOnly>
{
public override void SetValue(IDbDataParameter parameter, DateOnly value)
{
parameter.Value = value;
if (parameter is NpgsqlParameter npgsqlParameter)
{
npgsqlParameter.NpgsqlDbType = NpgsqlDbType.Date;
}
}
public override DateOnly Parse(object value)
{
return value switch
{
DateOnly dateOnly => dateOnly,
DateTime dateTime => DateOnly.FromDateTime(dateTime),
_ => DateOnly.FromDateTime((DateTime)value)
};
}
}
private sealed class NullableDateOnlyTypeHandler : SqlMapper.TypeHandler<DateOnly?>
{
public override void SetValue(IDbDataParameter parameter, DateOnly? value)
{
if (value is null)
{
parameter.Value = DBNull.Value;
return;
}
parameter.Value = value.Value;
if (parameter is NpgsqlParameter npgsqlParameter)
{
npgsqlParameter.NpgsqlDbType = NpgsqlDbType.Date;
}
}
public override DateOnly? Parse(object value)
{
if (value is null || value is DBNull)
{
return null;
}
return value switch
{
DateOnly dateOnly => dateOnly,
DateTime dateTime => DateOnly.FromDateTime(dateTime),
_ => DateOnly.FromDateTime((DateTime)value)
};
}
}
}

277
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresEpssSignalRepository.cs
View File

@@ -6,7 +6,8 @@
// -----------------------------------------------------------------------------
using Dapper;
using Microsoft.EntityFrameworkCore;
using Npgsql;
using StellaOps.Scanner.Storage.Repositories;
using System.Text.Json;
@@ -39,9 +40,9 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
is_model_change, model_version, dedupe_key, explain_hash, payload
)
VALUES (
@TenantId, @ModelDate, @CveId, @EventType, @RiskBand,
@EpssScore, @EpssDelta, @Percentile, @PercentileDelta,
@IsModelChange, @ModelVersion, @DedupeKey, @ExplainHash, @Payload::jsonb
$1, $2, $3, $4, $5,
$6, $7, $8, $9,
$10, $11, $12, $13, $14::jsonb
)
ON CONFLICT (tenant_id, dedupe_key) DO NOTHING
RETURNING signal_id, created_at
@@ -49,30 +50,31 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
await using var connection = await _dataSource.OpenConnectionAsync(signal.TenantId.ToString("D"), cancellationToken);
var result = await connection.QueryFirstOrDefaultAsync<(long signal_id, DateTimeOffset created_at)?>(sql, new
{
signal.TenantId,
ModelDate = signal.ModelDate.ToDateTime(TimeOnly.MinValue),
signal.CveId,
signal.EventType,
signal.RiskBand,
signal.EpssScore,
signal.EpssDelta,
signal.Percentile,
signal.PercentileDelta,
signal.IsModelChange,
signal.ModelVersion,
signal.DedupeKey,
signal.ExplainHash,
signal.Payload
});
await using var cmd = new NpgsqlCommand(sql, connection);
cmd.Parameters.AddWithValue(signal.TenantId);
cmd.Parameters.AddWithValue(signal.ModelDate.ToDateTime(TimeOnly.MinValue));
cmd.Parameters.AddWithValue(signal.CveId);
cmd.Parameters.AddWithValue(signal.EventType);
cmd.Parameters.AddWithValue((object?)signal.RiskBand ?? DBNull.Value);
cmd.Parameters.AddWithValue(signal.EpssScore.HasValue ? signal.EpssScore.Value : DBNull.Value);
cmd.Parameters.AddWithValue(signal.EpssDelta.HasValue ? signal.EpssDelta.Value : DBNull.Value);
cmd.Parameters.AddWithValue(signal.Percentile.HasValue ? signal.Percentile.Value : DBNull.Value);
cmd.Parameters.AddWithValue(signal.PercentileDelta.HasValue ? signal.PercentileDelta.Value : DBNull.Value);
cmd.Parameters.AddWithValue(signal.IsModelChange);
cmd.Parameters.AddWithValue((object?)signal.ModelVersion ?? DBNull.Value);
cmd.Parameters.AddWithValue(signal.DedupeKey);
cmd.Parameters.AddWithValue(signal.ExplainHash);
cmd.Parameters.AddWithValue(signal.Payload);
if (result.HasValue)
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
{
var signalId = reader.GetInt64(0);
var createdAt = reader.GetFieldValue<DateTimeOffset>(1);
return signal with
{
SignalId = result.Value.signal_id,
CreatedAt = result.Value.created_at
SignalId = signalId,
CreatedAt = createdAt
};
}
@@ -98,9 +100,9 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
is_model_change, model_version, dedupe_key, explain_hash, payload
)
VALUES (
@TenantId, @ModelDate, @CveId, @EventType, @RiskBand,
@EpssScore, @EpssDelta, @Percentile, @PercentileDelta,
@IsModelChange, @ModelVersion, @DedupeKey, @ExplainHash, @Payload::jsonb
$1, $2, $3, $4, $5,
$6, $7, $8, $9,
$10, $11, $12, $13, $14::jsonb
)
ON CONFLICT (tenant_id, dedupe_key) DO NOTHING
""";
@@ -113,25 +115,23 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
foreach (var signal in tenantGroup)
{
var affected = await connection.ExecuteAsync(sql, new
{
signal.TenantId,
ModelDate = signal.ModelDate.ToDateTime(TimeOnly.MinValue),
signal.CveId,
signal.EventType,
signal.RiskBand,
signal.EpssScore,
signal.EpssDelta,
signal.Percentile,
signal.PercentileDelta,
signal.IsModelChange,
signal.ModelVersion,
signal.DedupeKey,
signal.ExplainHash,
signal.Payload
}, transaction);
await using var cmd = new NpgsqlCommand(sql, connection, transaction);
cmd.Parameters.AddWithValue(signal.TenantId);
cmd.Parameters.AddWithValue(signal.ModelDate.ToDateTime(TimeOnly.MinValue));
cmd.Parameters.AddWithValue(signal.CveId);
cmd.Parameters.AddWithValue(signal.EventType);
cmd.Parameters.AddWithValue((object?)signal.RiskBand ?? DBNull.Value);
cmd.Parameters.AddWithValue(signal.EpssScore.HasValue ? signal.EpssScore.Value : DBNull.Value);
cmd.Parameters.AddWithValue(signal.EpssDelta.HasValue ? signal.EpssDelta.Value : DBNull.Value);
cmd.Parameters.AddWithValue(signal.Percentile.HasValue ? signal.Percentile.Value : DBNull.Value);
cmd.Parameters.AddWithValue(signal.PercentileDelta.HasValue ? signal.PercentileDelta.Value : DBNull.Value);
cmd.Parameters.AddWithValue(signal.IsModelChange);
cmd.Parameters.AddWithValue((object?)signal.ModelVersion ?? DBNull.Value);
cmd.Parameters.AddWithValue(signal.DedupeKey);
cmd.Parameters.AddWithValue(signal.ExplainHash);
cmd.Parameters.AddWithValue(signal.Payload);
inserted += affected;
inserted += await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
await transaction.CommitAsync(cancellationToken);
@@ -150,29 +150,42 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
var eventTypeList = eventTypes?.ToList();
var hasEventTypeFilter = eventTypeList?.Count > 0;
var paramList = new List<object>
{
tenantId,
startDate.ToDateTime(TimeOnly.MinValue),
endDate.ToDateTime(TimeOnly.MinValue)
};
var paramIndex = 3;
var eventTypeClause = "";
if (hasEventTypeFilter)
{
eventTypeClause = $"AND event_type = ANY(@p{paramIndex})";
paramList.Add(eventTypeList!.ToArray());
}
var sql = $"""
SELECT
signal_id, tenant_id, model_date, cve_id, event_type, risk_band,
epss_score, epss_delta, percentile, percentile_delta,
is_model_change, model_version, dedupe_key, explain_hash, payload, created_at
FROM {SignalTable}
WHERE tenant_id = @TenantId
AND model_date >= @StartDate
AND model_date <= @EndDate
{(hasEventTypeFilter ? "AND event_type = ANY(@EventTypes)" : "")}
WHERE tenant_id = @p0
AND model_date >= @p1
AND model_date <= @p2
{eventTypeClause}
ORDER BY model_date DESC, created_at DESC
LIMIT 10000
""";
await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await connection.QueryAsync<SignalRow>(sql, new
{
TenantId = tenantId,
StartDate = startDate.ToDateTime(TimeOnly.MinValue),
EndDate = endDate.ToDateTime(TimeOnly.MinValue),
EventTypes = eventTypeList?.ToArray()
});
var rows = await dbContext.Database.SqlQueryRaw<SignalRow>(
sql, paramList.ToArray())
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows.Select(MapToSignal).ToList();
}
@@ -189,20 +202,19 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
epss_score, epss_delta, percentile, percentile_delta,
is_model_change, model_version, dedupe_key, explain_hash, payload, created_at
FROM {SignalTable}
WHERE tenant_id = @TenantId
AND cve_id = @CveId
WHERE tenant_id = @p0
AND cve_id = @p1
ORDER BY model_date DESC, created_at DESC
LIMIT @Limit
LIMIT @p2
""";
await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await connection.QueryAsync<SignalRow>(sql, new
{
TenantId = tenantId,
CveId = cveId,
Limit = limit
});
var rows = await dbContext.Database.SqlQueryRaw<SignalRow>(
sql, tenantId, cveId, limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows.Select(MapToSignal).ToList();
}
@@ -219,22 +231,21 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
epss_score, epss_delta, percentile, percentile_delta,
is_model_change, model_version, dedupe_key, explain_hash, payload, created_at
FROM {SignalTable}
WHERE tenant_id = @TenantId
AND model_date >= @StartDate
AND model_date <= @EndDate
WHERE tenant_id = @p0
AND model_date >= @p1
AND model_date <= @p2
AND risk_band IN ('CRITICAL', 'HIGH')
ORDER BY model_date DESC, created_at DESC
LIMIT 10000
""";
await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await connection.QueryAsync<SignalRow>(sql, new
{
TenantId = tenantId,
StartDate = startDate.ToDateTime(TimeOnly.MinValue),
EndDate = endDate.ToDateTime(TimeOnly.MinValue)
});
var rows = await dbContext.Database.SqlQueryRaw<SignalRow>(
sql, tenantId, startDate.ToDateTime(TimeOnly.MinValue), endDate.ToDateTime(TimeOnly.MinValue))
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return rows.Select(MapToSignal).ToList();
}
@@ -248,14 +259,18 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
big_jump_delta, suppress_on_model_change, enabled_event_types,
created_at, updated_at
FROM {ConfigTable}
WHERE tenant_id = @TenantId
WHERE tenant_id = @p0
""";
await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var row = await connection.QueryFirstOrDefaultAsync<ConfigRow?>(sql, new { TenantId = tenantId });
var row = await dbContext.Database.SqlQueryRaw<ConfigRow>(
sql, tenantId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return row.HasValue ? MapToConfig(row.Value) : null;
return row is not null && row.config_id != Guid.Empty ? MapToConfig(row) : null;
}
public async Task<EpssSignalConfig> UpsertConfigAsync(EpssSignalConfig config, CancellationToken cancellationToken = default)
@@ -268,8 +283,7 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
big_jump_delta, suppress_on_model_change, enabled_event_types
)
VALUES (
@TenantId, @CriticalPercentile, @HighPercentile, @MediumPercentile,
@BigJumpDelta, @SuppressOnModelChange, @EnabledEventTypes
$1, $2, $3, $4, $5, $6, $7
)
ON CONFLICT (tenant_id) DO UPDATE SET
critical_percentile = EXCLUDED.critical_percentile,
@@ -284,31 +298,35 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
await using var connection = await _dataSource.OpenConnectionAsync(config.TenantId.ToString("D"), cancellationToken);
var result = await connection.QueryFirstAsync<(Guid config_id, DateTimeOffset created_at, DateTimeOffset updated_at)>(sql, new
{
config.TenantId,
config.CriticalPercentile,
config.HighPercentile,
config.MediumPercentile,
config.BigJumpDelta,
config.SuppressOnModelChange,
EnabledEventTypes = config.EnabledEventTypes.ToArray()
});
await using var cmd = new NpgsqlCommand(sql, connection);
cmd.Parameters.AddWithValue(config.TenantId);
cmd.Parameters.AddWithValue(config.CriticalPercentile);
cmd.Parameters.AddWithValue(config.HighPercentile);
cmd.Parameters.AddWithValue(config.MediumPercentile);
cmd.Parameters.AddWithValue(config.BigJumpDelta);
cmd.Parameters.AddWithValue(config.SuppressOnModelChange);
cmd.Parameters.AddWithValue(config.EnabledEventTypes.ToArray());
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
await reader.ReadAsync(cancellationToken).ConfigureAwait(false);
return config with
{
ConfigId = result.config_id,
CreatedAt = result.created_at,
UpdatedAt = result.updated_at
ConfigId = reader.GetGuid(0),
CreatedAt = reader.GetFieldValue<DateTimeOffset>(1),
UpdatedAt = reader.GetFieldValue<DateTimeOffset>(2)
};
}
public async Task<int> PruneAsync(int retentionDays = 90, CancellationToken cancellationToken = default)
{
var sql = $"SELECT {SchemaName}.prune_epss_signals(@RetentionDays)";
var sql = $"SELECT {SchemaName}.prune_epss_signals($1)";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
return await connection.ExecuteScalarAsync<int>(sql, new { RetentionDays = retentionDays });
await using var cmd = new NpgsqlCommand(sql, connection);
cmd.Parameters.AddWithValue(retentionDays);
var result = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
return Convert.ToInt32(result);
}
private async Task<EpssSignal?> GetByDedupeKeyAsync(Guid tenantId, string dedupeKey, CancellationToken cancellationToken)
@@ -319,13 +337,18 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
epss_score, epss_delta, percentile, percentile_delta,
is_model_change, model_version, dedupe_key, explain_hash, payload, created_at
FROM {SignalTable}
WHERE tenant_id = @TenantId AND dedupe_key = @DedupeKey
WHERE tenant_id = @p0 AND dedupe_key = @p1
""";
await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken);
var row = await connection.QueryFirstOrDefaultAsync<SignalRow?>(sql, new { TenantId = tenantId, DedupeKey = dedupeKey });
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return row.HasValue ? MapToSignal(row.Value) : null;
var row = await dbContext.Database.SqlQueryRaw<SignalRow>(
sql, tenantId, dedupeKey)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return row is not null && row.signal_id != 0 ? MapToSignal(row) : null;
}
private static EpssSignal MapToSignal(SignalRow row)
@@ -368,33 +391,37 @@ public sealed class PostgresEpssSignalRepository : IEpssSignalRepository
};
}
private readonly record struct SignalRow(
long signal_id,
Guid tenant_id,
DateOnly model_date,
string cve_id,
string event_type,
string? risk_band,
double? epss_score,
double? epss_delta,
double? percentile,
double? percentile_delta,
bool is_model_change,
string? model_version,
string dedupe_key,
byte[] explain_hash,
string payload,
DateTimeOffset created_at);
private sealed class SignalRow
{
public long signal_id { get; set; }
public Guid tenant_id { get; set; }
public DateOnly model_date { get; set; }
public string cve_id { get; set; } = "";
public string event_type { get; set; } = "";
public string? risk_band { get; set; }
public double? epss_score { get; set; }
public double? epss_delta { get; set; }
public double? percentile { get; set; }
public double? percentile_delta { get; set; }
public bool is_model_change { get; set; }
public string? model_version { get; set; }
public string dedupe_key { get; set; } = "";
public byte[] explain_hash { get; set; } = [];
public string payload { get; set; } = "";
public DateTimeOffset created_at { get; set; }
}
private readonly record struct ConfigRow(
Guid config_id,
Guid tenant_id,
double critical_percentile,
double high_percentile,
double medium_percentile,
double big_jump_delta,
bool suppress_on_model_change,
string[]? enabled_event_types,
DateTimeOffset created_at,
DateTimeOffset updated_at);
private sealed class ConfigRow
{
public Guid config_id { get; set; }
public Guid tenant_id { get; set; }
public double critical_percentile { get; set; }
public double high_percentile { get; set; }
public double medium_percentile { get; set; }
public double big_jump_delta { get; set; }
public bool suppress_on_model_change { get; set; }
public string[]? enabled_event_types { get; set; }
public DateTimeOffset created_at { get; set; }
public DateTimeOffset updated_at { get; set; }
}
}

95
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresIdempotencyKeyRepository.cs
View File

@@ -3,12 +3,13 @@
// Sprint: SPRINT_3500_0002_0003_proof_replay_api
// Task: T3 - Idempotency Middleware
// Description: PostgreSQL implementation of idempotency key repository
// Converted from Dapper to EF Core; ON CONFLICT upsert and stored function kept as raw SQL.
// -----------------------------------------------------------------------------
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Determinism;
using StellaOps.Scanner.Storage.EfCore.Models;
using StellaOps.Scanner.Storage.Entities;
using StellaOps.Scanner.Storage.Repositories;
@@ -16,6 +17,7 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// PostgreSQL implementation of <see cref="IIdempotencyKeyRepository"/>.
/// Converted from Dapper to EF Core; ON CONFLICT upsert and stored function kept as raw SQL.
/// </summary>
public sealed class PostgresIdempotencyKeyRepository : IIdempotencyKeyRepository
{
@@ -41,28 +43,20 @@ public sealed class PostgresIdempotencyKeyRepository : IIdempotencyKeyRepository
string endpointPath,
CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
key_id AS KeyId,
tenant_id AS TenantId,
content_digest AS ContentDigest,
endpoint_path AS EndpointPath,
response_status AS ResponseStatus,
response_body AS ResponseBody,
response_headers AS ResponseHeaders,
created_at AS CreatedAt,
expires_at AS ExpiresAt
FROM {SchemaName}.idempotency_keys
WHERE tenant_id = @TenantId
AND content_digest = @ContentDigest
AND endpoint_path = @EndpointPath
AND expires_at > now()
""";
await using var conn = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await conn.QuerySingleOrDefaultAsync<IdempotencyKeyRow>(
new CommandDefinition(sql, new { TenantId = tenantId, ContentDigest = contentDigest, EndpointPath = endpointPath }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(conn, _dataSource.CommandTimeoutSeconds, SchemaName);
var now = DateTimeOffset.UtcNow;
var entity = await dbContext.IdempotencyKeys
.AsNoTracking()
.Where(e => e.TenantId == tenantId
&& e.ContentDigest == contentDigest
&& e.EndpointPath == endpointPath
&& e.ExpiresAt > now)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapToRow(entity);
}
/// <inheritdoc />
@@ -75,15 +69,16 @@ public sealed class PostgresIdempotencyKeyRepository : IIdempotencyKeyRepository
key.KeyId = _guidProvider.NewGuid();
}
// Keep raw SQL for ON CONFLICT upsert + jsonb casts.
var sql = $"""
INSERT INTO {SchemaName}.idempotency_keys
(key_id, tenant_id, content_digest, endpoint_path,
response_status, response_body, response_headers,
created_at, expires_at)
VALUES
(@KeyId, @TenantId, @ContentDigest, @EndpointPath,
@ResponseStatus, @ResponseBody::jsonb, @ResponseHeaders::jsonb,
@CreatedAt, @ExpiresAt)
(@p0, @p1, @p2, @p3,
@p4, @p5::jsonb, @p6::jsonb,
@p7, @p8)
ON CONFLICT (tenant_id, content_digest, endpoint_path) DO UPDATE
SET response_status = EXCLUDED.response_status,
response_body = EXCLUDED.response_body,
@@ -94,22 +89,19 @@ public sealed class PostgresIdempotencyKeyRepository : IIdempotencyKeyRepository
""";
await using var conn = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var keyId = await conn.ExecuteScalarAsync<Guid>(
new CommandDefinition(sql, new
{
key.KeyId,
key.TenantId,
key.ContentDigest,
key.EndpointPath,
key.ResponseStatus,
key.ResponseBody,
key.ResponseHeaders,
key.CreatedAt,
key.ExpiresAt
}, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(conn, _dataSource.CommandTimeoutSeconds, SchemaName);
var result = await dbContext.Database.SqlQueryRaw<IdempotencyKeyInsertResult>(
sql,
key.KeyId, key.TenantId, key.ContentDigest, key.EndpointPath,
key.ResponseStatus,
(object?)key.ResponseBody ?? DBNull.Value,
(object?)key.ResponseHeaders ?? DBNull.Value,
key.CreatedAt, key.ExpiresAt)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
key.KeyId = keyId;
key.KeyId = result.key_id;
_logger.LogDebug(
"Saved idempotency key {KeyId} for tenant {TenantId}, digest {ContentDigest}",
@@ -121,11 +113,14 @@ public sealed class PostgresIdempotencyKeyRepository : IIdempotencyKeyRepository
/// <inheritdoc />
public async Task<int> DeleteExpiredAsync(CancellationToken cancellationToken = default)
{
// Keep raw SQL for stored function call.
var sql = $"SELECT {SchemaName}.cleanup_expired_idempotency_keys()";
await using var conn = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var result = await conn.ExecuteScalarAsync<int>(
new CommandDefinition(sql, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(conn, _dataSource.CommandTimeoutSeconds, SchemaName);
var result = await dbContext.Database.SqlQueryRaw<int>(sql)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
if (result > 0)
@@ -135,4 +130,22 @@ public sealed class PostgresIdempotencyKeyRepository : IIdempotencyKeyRepository
return result;
}
private static IdempotencyKeyRow MapToRow(IdempotencyKeyEntity e) => new()
{
KeyId = e.KeyId,
TenantId = e.TenantId,
ContentDigest = e.ContentDigest,
EndpointPath = e.EndpointPath,
ResponseStatus = e.ResponseStatus,
ResponseBody = e.ResponseBody,
ResponseHeaders = e.ResponseHeaders,
CreatedAt = e.CreatedAt,
ExpiresAt = e.ExpiresAt
};
private sealed record IdempotencyKeyInsertResult
{
public Guid key_id { get; init; }
}
}

160
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresMaterialRiskChangeRepository.cs
View File

@@ -1,5 +1,5 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Scanner.SmartDiff.Detection;
@@ -14,9 +14,6 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// </summary>
public sealed class PostgresMaterialRiskChangeRepository : IMaterialRiskChangeRepository
{
private const string TenantContext = "00000000-0000-0000-0000-000000000001";
private static readonly Guid TenantId = Guid.Parse(TenantContext);
private readonly ScannerDataSource _dataSource;
private readonly ILogger<PostgresMaterialRiskChangeRepository> _logger;
@@ -36,31 +33,41 @@ public sealed class PostgresMaterialRiskChangeRepository : IMaterialRiskChangeRe
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task StoreChangeAsync(MaterialRiskChangeResult change, string scanId, CancellationToken ct = default)
public async Task StoreChangeAsync(
MaterialRiskChangeResult change,
string scanId,
CancellationToken ct = default,
string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(change);
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
await InsertChangeAsync(connection, change, scanId.Trim(), ct).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await InsertChangeAsync(connection, change, scanId.Trim(), tenantScope.TenantId, ct).ConfigureAwait(false);
}
public async Task StoreChangesAsync(IReadOnlyList<MaterialRiskChangeResult> changes, string scanId, CancellationToken ct = default)
public async Task StoreChangesAsync(
IReadOnlyList<MaterialRiskChangeResult> changes,
string scanId,
CancellationToken ct = default,
string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(changes);
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
if (changes.Count == 0)
return;
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false);
try
{
foreach (var change in changes)
{
await InsertChangeAsync(connection, change, scanId.Trim(), ct, transaction).ConfigureAwait(false);
await InsertChangeAsync(connection, change, scanId.Trim(), tenantScope.TenantId, ct, transaction).ConfigureAwait(false);
}
await transaction.CommitAsync(ct).ConfigureAwait(false);
@@ -74,22 +81,31 @@ public sealed class PostgresMaterialRiskChangeRepository : IMaterialRiskChangeRe
}
}
public async Task<IReadOnlyList<MaterialRiskChangeResult>> GetChangesForScanAsync(string scanId, CancellationToken ct = default)
public async Task<IReadOnlyList<MaterialRiskChangeResult>> GetChangesForScanAsync(
string scanId,
CancellationToken ct = default,
string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
SELECT
vuln_id, purl, has_material_change, priority_score,
previous_state_hash, current_state_hash, changes
FROM {MaterialRiskChangesTable}
WHERE tenant_id = @TenantId
AND scan_id = @ScanId
WHERE tenant_id = @p0
AND scan_id = @p1
ORDER BY priority_score DESC
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var rows = await connection.QueryAsync<MaterialRiskChangeRow>(sql, new { TenantId, ScanId = scanId.Trim() });
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<MaterialRiskChangeRow>(
sql, tenantScope.TenantId, scanId.Trim())
.ToListAsync(ct)
.ConfigureAwait(false);
return rows.Select(r => r.ToResult()).ToList();
}
@@ -97,94 +113,99 @@ public sealed class PostgresMaterialRiskChangeRepository : IMaterialRiskChangeRe
public async Task<IReadOnlyList<MaterialRiskChangeResult>> GetChangesForFindingAsync(
FindingKey findingKey,
int limit = 10,
CancellationToken ct = default)
CancellationToken ct = default,
string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(findingKey);
ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
SELECT
vuln_id, purl, has_material_change, priority_score,
previous_state_hash, current_state_hash, changes
FROM {MaterialRiskChangesTable}
WHERE tenant_id = @TenantId
AND vuln_id = @VulnId
AND purl = @Purl
WHERE tenant_id = @p0
AND vuln_id = @p1
AND purl = @p2
ORDER BY detected_at DESC
LIMIT @Limit
LIMIT @p3
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var rows = await connection.QueryAsync<MaterialRiskChangeRow>(sql, new
{
TenantId,
VulnId = findingKey.VulnId,
Purl = findingKey.ComponentPurl,
Limit = limit
});
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<MaterialRiskChangeRow>(
sql, tenantScope.TenantId, findingKey.VulnId, findingKey.ComponentPurl, limit)
.ToListAsync(ct)
.ConfigureAwait(false);
return rows.Select(r => r.ToResult()).ToList();
}
public async Task<MaterialRiskChangeQueryResult> QueryChangesAsync(
MaterialRiskChangeQuery query,
CancellationToken ct = default)
CancellationToken ct = default,
string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(query);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var conditions = new List<string> { "has_material_change = TRUE" };
var parameters = new DynamicParameters();
if (!string.IsNullOrEmpty(query.ImageDigest))
{
// Would need a join with scan metadata for image filtering
// For now, skip this filter
}
var paramList = new List<object>();
var paramIndex = 0;
if (query.Since.HasValue)
{
conditions.Add("detected_at >= @Since");
parameters.Add("Since", query.Since.Value);
conditions.Add($"detected_at >= @p{paramIndex}");
paramList.Add(query.Since.Value);
paramIndex++;
}
if (query.Until.HasValue)
{
conditions.Add("detected_at <= @Until");
parameters.Add("Until", query.Until.Value);
conditions.Add($"detected_at <= @p{paramIndex}");
paramList.Add(query.Until.Value);
paramIndex++;
}
if (query.MinPriorityScore.HasValue)
{
conditions.Add("priority_score >= @MinPriority");
parameters.Add("MinPriority", query.MinPriorityScore.Value);
conditions.Add($"priority_score >= @p{paramIndex}");
paramList.Add(query.MinPriorityScore.Value);
paramIndex++;
}
conditions.Add("tenant_id = @TenantId");
parameters.Add("TenantId", TenantId);
conditions.Add($"tenant_id = @p{paramIndex}");
paramList.Add(tenantScope.TenantId);
paramIndex++;
var whereClause = string.Join(" AND ", conditions);
// Count query
var countSql = $"SELECT COUNT(*) FROM {MaterialRiskChangesTable} WHERE {whereClause}";
// Data query
var dataSql = $"""
SELECT
SELECT
vuln_id, purl, has_material_change, priority_score,
previous_state_hash, current_state_hash, changes
FROM {MaterialRiskChangesTable}
WHERE {whereClause}
ORDER BY priority_score DESC
OFFSET @Offset LIMIT @Limit
OFFSET @p{paramIndex} LIMIT @p{paramIndex + 1}
""";
parameters.Add("Offset", query.Offset);
parameters.Add("Limit", query.Limit);
var dataParams = new List<object>(paramList) { query.Offset, query.Limit };
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var totalCount = await connection.ExecuteScalarAsync<int>(countSql, parameters);
var rows = await connection.QueryAsync<MaterialRiskChangeRow>(dataSql, parameters);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var totalCount = await dbContext.Database.SqlQueryRaw<int>(countSql, paramList.ToArray())
.FirstAsync(ct).ConfigureAwait(false);
var rows = await dbContext.Database.SqlQueryRaw<MaterialRiskChangeRow>(dataSql, dataParams.ToArray())
.ToListAsync(ct).ConfigureAwait(false);
var changes = rows.Select(r => r.ToResult()).ToImmutableArray();
@@ -199,6 +220,7 @@ public sealed class PostgresMaterialRiskChangeRepository : IMaterialRiskChangeRe
NpgsqlConnection connection,
MaterialRiskChangeResult change,
string scanId,
Guid tenantId,
CancellationToken ct,
NpgsqlTransaction? transaction = null)
{
@@ -212,9 +234,7 @@ public sealed class PostgresMaterialRiskChangeRepository : IMaterialRiskChangeRe
has_material_change, priority_score,
previous_state_hash, current_state_hash, changes
) VALUES (
@TenantId, @VulnId, @Purl, @ScanId,
@HasMaterialChange, @PriorityScore,
@PreviousStateHash, @CurrentStateHash, @Changes::jsonb
$1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb
)
ON CONFLICT (tenant_id, scan_id, vuln_id, purl) DO UPDATE SET
has_material_change = EXCLUDED.has_material_change,
@@ -226,18 +246,18 @@ public sealed class PostgresMaterialRiskChangeRepository : IMaterialRiskChangeRe
var changesJson = JsonSerializer.Serialize(change.Changes, JsonOptions);
await connection.ExecuteAsync(new CommandDefinition(sql, new
{
TenantId,
VulnId = change.FindingKey.VulnId,
Purl = change.FindingKey.ComponentPurl,
ScanId = scanId,
HasMaterialChange = change.HasMaterialChange,
PriorityScore = change.PriorityScore,
PreviousStateHash = change.PreviousStateHash,
CurrentStateHash = change.CurrentStateHash,
Changes = changesJson
}, transaction: transaction, cancellationToken: ct));
await using var cmd = new NpgsqlCommand(sql, connection, transaction);
cmd.Parameters.AddWithValue(tenantId);
cmd.Parameters.AddWithValue(change.FindingKey.VulnId);
cmd.Parameters.AddWithValue(change.FindingKey.ComponentPurl);
cmd.Parameters.AddWithValue(scanId);
cmd.Parameters.AddWithValue(change.HasMaterialChange);
cmd.Parameters.AddWithValue(change.PriorityScore);
cmd.Parameters.AddWithValue(change.PreviousStateHash);
cmd.Parameters.AddWithValue(change.CurrentStateHash);
cmd.Parameters.AddWithValue(changesJson);
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
/// <summary>

59
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresObservedCveRepository.cs
View File

@@ -3,9 +3,10 @@
// Sprint: SPRINT_3413_0001_0001_epss_live_enrichment
// Task: S6 - Add observed CVEs filter
// Description: PostgreSQL implementation of IObservedCveRepository.
// Converted from Dapper to EF Core raw SQL (triage table not modeled in Scanner DbContext).
// -----------------------------------------------------------------------------
using Dapper;
using Microsoft.EntityFrameworkCore;
using StellaOps.Scanner.Storage.Repositories;
namespace StellaOps.Scanner.Storage.Postgres;
@@ -13,6 +14,7 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// PostgreSQL implementation of <see cref="IObservedCveRepository"/>.
/// Queries vuln_instance_triage to determine which CVEs are observed per tenant.
/// Converted from Dapper to EF Core raw SQL.
/// </summary>
public sealed class PostgresObservedCveRepository : IObservedCveRepository
{
@@ -33,13 +35,17 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository
var sql = $"""
SELECT DISTINCT cve_id
FROM {TriageTable}
WHERE tenant_id = @TenantId
WHERE tenant_id = @p0
AND cve_id IS NOT NULL
AND cve_id LIKE 'CVE-%'
""";
await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken);
var cves = await connection.QueryAsync<string>(sql, new { TenantId = tenantId });
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var cves = await dbContext.Database.SqlQueryRaw<string>(sql, tenantId)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return new HashSet<string>(cves, StringComparer.OrdinalIgnoreCase);
}
@@ -52,13 +58,17 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository
var sql = $"""
SELECT EXISTS (
SELECT 1 FROM {TriageTable}
WHERE tenant_id = @TenantId
AND cve_id = @CveId
WHERE tenant_id = @p0
AND cve_id = @p1
)
""";
await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken);
return await connection.ExecuteScalarAsync<bool>(sql, new { TenantId = tenantId, CveId = cveId });
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return await dbContext.Database.SqlQueryRaw<bool>(sql, tenantId, cveId)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
}
public async Task<IReadOnlySet<string>> FilterObservedAsync(
@@ -75,16 +85,16 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository
var sql = $"""
SELECT DISTINCT cve_id
FROM {TriageTable}
WHERE tenant_id = @TenantId
AND cve_id = ANY(@CveIds)
WHERE tenant_id = @p0
AND cve_id = ANY(@p1)
""";
await using var connection = await _dataSource.OpenConnectionAsync(tenantId.ToString("D"), cancellationToken);
var observed = await connection.QueryAsync<string>(sql, new
{
TenantId = tenantId,
CveIds = cveList.ToArray()
});
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var observed = await dbContext.Database.SqlQueryRaw<string>(sql, tenantId, cveList.ToArray())
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return new HashSet<string>(observed, StringComparer.OrdinalIgnoreCase);
}
@@ -100,9 +110,11 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var tenants = await connection.QueryAsync<Guid>(sql);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return tenants.ToList();
return await dbContext.Database.SqlQueryRaw<Guid>(sql)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
public async Task<IReadOnlyDictionary<string, IReadOnlyList<Guid>>> GetTenantsObservingCvesAsync(
@@ -118,15 +130,16 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository
var sql = $"""
SELECT cve_id, tenant_id
FROM {TriageTable}
WHERE cve_id = ANY(@CveIds)
WHERE cve_id = ANY(@p0)
GROUP BY cve_id, tenant_id
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
var rows = await connection.QueryAsync<(string cve_id, Guid tenant_id)>(sql, new
{
CveIds = cveList.ToArray()
});
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<CveTenantRow>(sql, (object)cveList.ToArray())
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var result = new Dictionary<string, List<Guid>>(StringComparer.OrdinalIgnoreCase);
@@ -149,4 +162,10 @@ public sealed class PostgresObservedCveRepository : IObservedCveRepository
kvp => (IReadOnlyList<Guid>)kvp.Value,
StringComparer.OrdinalIgnoreCase);
}
private sealed record CveTenantRow
{
public string cve_id { get; init; } = "";
public Guid tenant_id { get; init; }
}
}

141
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresProofBundleRepository.cs
View File

@@ -1,4 +1,5 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using StellaOps.Scanner.Storage.EfCore.Models;
using StellaOps.Scanner.Storage.Entities;
using StellaOps.Scanner.Storage.Repositories;
@@ -6,12 +7,12 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// PostgreSQL implementation of proof bundle repository.
/// Converted from Dapper to EF Core; ON CONFLICT upsert kept as raw SQL.
/// </summary>
public sealed class PostgresProofBundleRepository : IProofBundleRepository
{
private readonly ScannerDataSource _dataSource;
private string SchemaName => _dataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private string TableName => $"{SchemaName}.proof_bundle";
public PostgresProofBundleRepository(ScannerDataSource dataSource)
{
@@ -20,68 +21,39 @@ public sealed class PostgresProofBundleRepository : IProofBundleRepository
public async Task<ProofBundleRow?> GetByRootHashAsync(string rootHash, CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
scan_id AS ScanId,
root_hash AS RootHash,
bundle_type AS BundleType,
dsse_envelope AS DsseEnvelope,
signature_keyid AS SignatureKeyId,
signature_algorithm AS SignatureAlgorithm,
bundle_content AS BundleContent,
bundle_hash AS BundleHash,
ledger_hash AS LedgerHash,
manifest_hash AS ManifestHash,
sbom_hash AS SbomHash,
vex_hash AS VexHash,
created_at AS CreatedAt,
expires_at AS ExpiresAt
FROM {TableName}
WHERE root_hash = @RootHash
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.QuerySingleOrDefaultAsync<ProofBundleRow>(
new CommandDefinition(sql, new { RootHash = rootHash }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entity = await dbContext.ProofBundles
.AsNoTracking()
.FirstOrDefaultAsync(e => e.RootHash == rootHash, cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapToRow(entity);
}
public async Task<IReadOnlyList<ProofBundleRow>> GetByScanIdAsync(Guid scanId, CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
scan_id AS ScanId,
root_hash AS RootHash,
bundle_type AS BundleType,
dsse_envelope AS DsseEnvelope,
signature_keyid AS SignatureKeyId,
signature_algorithm AS SignatureAlgorithm,
bundle_content AS BundleContent,
bundle_hash AS BundleHash,
ledger_hash AS LedgerHash,
manifest_hash AS ManifestHash,
sbom_hash AS SbomHash,
vex_hash AS VexHash,
created_at AS CreatedAt,
expires_at AS ExpiresAt
FROM {TableName}
WHERE scan_id = @ScanId
ORDER BY created_at DESC
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var results = await connection.QueryAsync<ProofBundleRow>(
new CommandDefinition(sql, new { ScanId = scanId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entities = await dbContext.ProofBundles
.AsNoTracking()
.Where(e => e.ScanId == scanId)
.OrderByDescending(e => e.CreatedAt)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return results.ToList();
return entities.Select(MapToRow).ToList();
}
public async Task<ProofBundleRow> SaveAsync(ProofBundleRow bundle, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(bundle);
// Keep raw SQL for ON CONFLICT upsert + jsonb cast.
var sql = $"""
INSERT INTO {TableName} (
INSERT INTO {SchemaName}.proof_bundle (
scan_id,
root_hash,
bundle_type,
@@ -96,47 +68,70 @@ public sealed class PostgresProofBundleRepository : IProofBundleRepository
vex_hash,
expires_at
) VALUES (
@ScanId,
@RootHash,
@BundleType,
@DsseEnvelope::jsonb,
@SignatureKeyId,
@SignatureAlgorithm,
@BundleContent,
@BundleHash,
@LedgerHash,
@ManifestHash,
@SbomHash,
@VexHash,
@ExpiresAt
@p0, @p1, @p2, @p3::jsonb, @p4, @p5, @p6, @p7, @p8, @p9, @p10, @p11, @p12
)
ON CONFLICT (scan_id, root_hash) DO UPDATE SET
dsse_envelope = EXCLUDED.dsse_envelope,
bundle_content = EXCLUDED.bundle_content,
bundle_hash = EXCLUDED.bundle_hash,
ledger_hash = EXCLUDED.ledger_hash
RETURNING created_at AS CreatedAt
RETURNING created_at
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var createdAt = await connection.QuerySingleAsync<DateTimeOffset>(
new CommandDefinition(sql, bundle, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var result = await dbContext.Database.SqlQueryRaw<ProofBundleInsertResult>(
sql,
bundle.ScanId, bundle.RootHash, bundle.BundleType,
(object?)bundle.DsseEnvelope ?? DBNull.Value,
(object?)bundle.SignatureKeyId ?? DBNull.Value,
(object?)bundle.SignatureAlgorithm ?? DBNull.Value,
(object?)bundle.BundleContent ?? DBNull.Value,
(object?)bundle.BundleHash ?? DBNull.Value,
(object?)bundle.LedgerHash ?? DBNull.Value,
(object?)bundle.ManifestHash ?? DBNull.Value,
(object?)bundle.SbomHash ?? DBNull.Value,
(object?)bundle.VexHash ?? DBNull.Value,
(object?)bundle.ExpiresAt ?? DBNull.Value)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
bundle.CreatedAt = createdAt;
bundle.CreatedAt = result.created_at;
return bundle;
}
public async Task<int> DeleteExpiredAsync(CancellationToken cancellationToken = default)
{
var sql = $"""
DELETE FROM {TableName}
WHERE expires_at IS NOT NULL AND expires_at < NOW()
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.ExecuteAsync(
new CommandDefinition(sql, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return await dbContext.ProofBundles
.Where(e => e.ExpiresAt != null && e.ExpiresAt < DateTimeOffset.UtcNow)
.ExecuteDeleteAsync(cancellationToken)
.ConfigureAwait(false);
}
private static ProofBundleRow MapToRow(ProofBundleEntity e) => new()
{
ScanId = e.ScanId,
RootHash = e.RootHash,
BundleType = e.BundleType,
DsseEnvelope = e.DsseEnvelope,
SignatureKeyId = e.SignatureKeyId,
SignatureAlgorithm = e.SignatureAlgorithm,
BundleContent = e.BundleContent,
BundleHash = e.BundleHash,
LedgerHash = e.LedgerHash,
ManifestHash = e.ManifestHash,
SbomHash = e.SbomHash,
VexHash = e.VexHash,
CreatedAt = e.CreatedAt,
ExpiresAt = e.ExpiresAt
};
private sealed record ProofBundleInsertResult
{
public DateTimeOffset created_at { get; init; }
}
}

298
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresReachabilityDriftResultRepository.cs
View File

@@ -1,6 +1,7 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Scanner.Contracts;
using StellaOps.Scanner.ReachabilityDrift;
using StellaOps.Scanner.Storage.Repositories;
@@ -11,9 +12,6 @@ namespace StellaOps.Scanner.Storage.Postgres;
public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDriftResultRepository
{
private const string TenantContext = "00000000-0000-0000-0000-000000000001";
private static readonly Guid TenantId = Guid.Parse(TenantContext);
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false
@@ -34,31 +32,17 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task StoreAsync(ReachabilityDriftResult result, CancellationToken ct = default)
public async Task StoreAsync(ReachabilityDriftResult result, CancellationToken ct = default, string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(result);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var insertResultSql = $"""
INSERT INTO {DriftResultsTable} (
id,
tenant_id,
base_scan_id,
head_scan_id,
language,
newly_reachable_count,
newly_unreachable_count,
detected_at,
result_digest
id, tenant_id, base_scan_id, head_scan_id, language,
newly_reachable_count, newly_unreachable_count, detected_at, result_digest
) VALUES (
@Id,
@TenantId,
@BaseScanId,
@HeadScanId,
@Language,
@NewlyReachableCount,
@NewlyUnreachableCount,
@DetectedAt,
@ResultDigest
$1, $2, $3, $4, $5, $6, $7, $8, $9
)
ON CONFLICT (tenant_id, base_scan_id, head_scan_id, language, result_digest) DO UPDATE SET
newly_reachable_count = EXCLUDED.newly_reachable_count,
@@ -69,42 +53,17 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
var deleteSinksSql = $"""
DELETE FROM {DriftedSinksTable}
WHERE tenant_id = @TenantId AND drift_result_id = @DriftId
WHERE tenant_id = $1 AND drift_result_id = $2
""";
var insertSinkSql = $"""
INSERT INTO {DriftedSinksTable} (
id,
tenant_id,
drift_result_id,
sink_node_id,
symbol,
sink_category,
direction,
cause_kind,
cause_description,
cause_symbol,
cause_file,
cause_line,
code_change_id,
compressed_path,
associated_vulns
id, tenant_id, drift_result_id, sink_node_id, symbol,
sink_category, direction, cause_kind, cause_description,
cause_symbol, cause_file, cause_line, code_change_id,
compressed_path, associated_vulns
) VALUES (
@Id,
@TenantId,
@DriftId,
@SinkNodeId,
@Symbol,
@SinkCategory,
@Direction,
@CauseKind,
@CauseDescription,
@CauseSymbol,
@CauseFile,
@CauseLine,
@CodeChangeId,
@CompressedPath::jsonb,
@AssociatedVulns::jsonb
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb, $15::jsonb
)
ON CONFLICT (drift_result_id, sink_node_id) DO UPDATE SET
symbol = EXCLUDED.symbol,
@@ -120,48 +79,57 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
associated_vulns = EXCLUDED.associated_vulns
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false);
try
{
var driftId = await connection.ExecuteScalarAsync<Guid>(new CommandDefinition(
insertResultSql,
new
{
result.Id,
TenantId,
BaseScanId = result.BaseScanId.Trim(),
HeadScanId = result.HeadScanId.Trim(),
Language = result.Language.Trim(),
NewlyReachableCount = result.NewlyReachable.Length,
NewlyUnreachableCount = result.NewlyUnreachable.Length,
DetectedAt = result.DetectedAt.UtcDateTime,
result.ResultDigest
},
transaction: transaction,
cancellationToken: ct))
.ConfigureAwait(false);
// Insert drift result header and get the returned id
await using var insertCmd = new NpgsqlCommand(insertResultSql, connection, transaction);
insertCmd.Parameters.AddWithValue(result.Id);
insertCmd.Parameters.AddWithValue(tenantScope.TenantId);
insertCmd.Parameters.AddWithValue(result.BaseScanId.Trim());
insertCmd.Parameters.AddWithValue(result.HeadScanId.Trim());
insertCmd.Parameters.AddWithValue(result.Language.Trim());
insertCmd.Parameters.AddWithValue(result.NewlyReachable.Length);
insertCmd.Parameters.AddWithValue(result.NewlyUnreachable.Length);
insertCmd.Parameters.AddWithValue(result.DetectedAt.UtcDateTime);
insertCmd.Parameters.AddWithValue(result.ResultDigest);
await connection.ExecuteAsync(new CommandDefinition(
deleteSinksSql,
new { TenantId, DriftId = driftId },
transaction: transaction,
cancellationToken: ct))
.ConfigureAwait(false);
var driftIdObj = await insertCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
var driftId = (Guid)driftIdObj!;
var sinkRows = EnumerateSinkRows(driftId, result.NewlyReachable, DriftDirection.BecameReachable)
.Concat(EnumerateSinkRows(driftId, result.NewlyUnreachable, DriftDirection.BecameUnreachable))
// Delete existing sinks for this drift result
await using var deleteCmd = new NpgsqlCommand(deleteSinksSql, connection, transaction);
deleteCmd.Parameters.AddWithValue(tenantScope.TenantId);
deleteCmd.Parameters.AddWithValue(driftId);
await deleteCmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
// Insert all sink rows
var sinks = EnumerateSinkParams(driftId, tenantScope.TenantId, result.NewlyReachable, DriftDirection.BecameReachable)
.Concat(EnumerateSinkParams(driftId, tenantScope.TenantId, result.NewlyUnreachable, DriftDirection.BecameUnreachable))
.ToList();
if (sinkRows.Count > 0)
foreach (var sink in sinks)
{
await connection.ExecuteAsync(new CommandDefinition(
insertSinkSql,
sinkRows,
transaction: transaction,
cancellationToken: ct))
.ConfigureAwait(false);
await using var sinkCmd = new NpgsqlCommand(insertSinkSql, connection, transaction);
sinkCmd.Parameters.AddWithValue(sink.Id);
sinkCmd.Parameters.AddWithValue(sink.TenantId);
sinkCmd.Parameters.AddWithValue(sink.DriftId);
sinkCmd.Parameters.AddWithValue(sink.SinkNodeId);
sinkCmd.Parameters.AddWithValue(sink.Symbol);
sinkCmd.Parameters.AddWithValue(sink.SinkCategory);
sinkCmd.Parameters.AddWithValue(sink.Direction);
sinkCmd.Parameters.AddWithValue(sink.CauseKind);
sinkCmd.Parameters.AddWithValue(sink.CauseDescription);
sinkCmd.Parameters.AddWithValue((object?)sink.CauseSymbol ?? DBNull.Value);
sinkCmd.Parameters.AddWithValue((object?)sink.CauseFile ?? DBNull.Value);
sinkCmd.Parameters.AddWithValue(sink.CauseLine.HasValue ? sink.CauseLine.Value : DBNull.Value);
sinkCmd.Parameters.AddWithValue(sink.CodeChangeId.HasValue ? sink.CodeChangeId.Value : DBNull.Value);
sinkCmd.Parameters.AddWithValue(sink.CompressedPath);
sinkCmd.Parameters.AddWithValue((object?)sink.AssociatedVulns ?? DBNull.Value);
await sinkCmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
await transaction.CommitAsync(ct).ConfigureAwait(false);
@@ -181,81 +149,81 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
}
}
public async Task<ReachabilityDriftResult?> TryGetLatestForHeadAsync(string headScanId, string language, CancellationToken ct = default)
public async Task<ReachabilityDriftResult?> TryGetLatestForHeadAsync(string headScanId, string language, CancellationToken ct = default, string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(headScanId);
ArgumentException.ThrowIfNullOrWhiteSpace(language);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT id, base_scan_id, head_scan_id, language, detected_at, result_digest
FROM {DriftResultsTable}
WHERE tenant_id = @TenantId AND head_scan_id = @HeadScanId AND language = @Language
WHERE tenant_id = @p0 AND head_scan_id = @p1 AND language = @p2
ORDER BY detected_at DESC
LIMIT 1
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var header = await connection.QuerySingleOrDefaultAsync<DriftHeaderRow>(new CommandDefinition(
sql,
new
{
TenantId,
HeadScanId = headScanId.Trim(),
Language = language.Trim()
},
cancellationToken: ct)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var header = await dbContext.Database.SqlQueryRaw<DriftHeaderRow>(
sql, tenantScope.TenantId, headScanId.Trim(), language.Trim())
.FirstOrDefaultAsync(ct)
.ConfigureAwait(false);
if (header is null)
{
return null;
}
return await LoadResultAsync(connection, header, ct).ConfigureAwait(false);
return await LoadResultAsync(connection, header, tenantScope.TenantId, ct).ConfigureAwait(false);
}
public async Task<ReachabilityDriftResult?> TryGetByIdAsync(Guid driftId, CancellationToken ct = default)
public async Task<ReachabilityDriftResult?> TryGetByIdAsync(Guid driftId, CancellationToken ct = default, string? tenantId = null)
{
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT id, base_scan_id, head_scan_id, language, detected_at, result_digest
FROM {DriftResultsTable}
WHERE tenant_id = @TenantId AND id = @DriftId
WHERE tenant_id = @p0 AND id = @p1
LIMIT 1
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var header = await connection.QuerySingleOrDefaultAsync<DriftHeaderRow>(new CommandDefinition(
sql,
new
{
TenantId,
DriftId = driftId
},
cancellationToken: ct)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var header = await dbContext.Database.SqlQueryRaw<DriftHeaderRow>(
sql, tenantScope.TenantId, driftId)
.FirstOrDefaultAsync(ct)
.ConfigureAwait(false);
if (header is null)
{
return null;
}
return await LoadResultAsync(connection, header, ct).ConfigureAwait(false);
return await LoadResultAsync(connection, header, tenantScope.TenantId, ct).ConfigureAwait(false);
}
public async Task<bool> ExistsAsync(Guid driftId, CancellationToken ct = default)
public async Task<bool> ExistsAsync(Guid driftId, CancellationToken ct = default, string? tenantId = null)
{
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT 1
SELECT CAST(1 AS integer) AS "Value"
FROM {DriftResultsTable}
WHERE tenant_id = @TenantId AND id = @DriftId
WHERE tenant_id = @p0 AND id = @p1
LIMIT 1
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var result = await connection.ExecuteScalarAsync<int?>(new CommandDefinition(
sql,
new { TenantId, DriftId = driftId },
cancellationToken: ct)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return result is not null;
var result = await dbContext.Database.SqlQueryRaw<int>(
sql, tenantScope.TenantId, driftId)
.FirstOrDefaultAsync(ct)
.ConfigureAwait(false);
return result != 0;
}
public async Task<IReadOnlyList<DriftedSink>> ListSinksAsync(
@@ -263,7 +231,8 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
DriftDirection direction,
int offset,
int limit,
CancellationToken ct = default)
CancellationToken ct = default,
string? tenantId = null)
{
if (offset < 0)
{
@@ -274,6 +243,7 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
{
throw new ArgumentOutOfRangeException(nameof(limit));
}
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
@@ -291,28 +261,27 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
compressed_path,
associated_vulns
FROM {DriftedSinksTable}
WHERE tenant_id = @TenantId AND drift_result_id = @DriftId AND direction = @Direction
WHERE tenant_id = @p0 AND drift_result_id = @p1 AND direction = @p2
ORDER BY sink_node_id ASC
OFFSET @Offset LIMIT @Limit
OFFSET @p3 LIMIT @p4
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var rows = await connection.QueryAsync<DriftSinkRow>(new CommandDefinition(
sql,
new
{
TenantId,
DriftId = driftId,
Direction = ToDbValue(direction),
Offset = offset,
Limit = limit
},
cancellationToken: ct)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<DriftSinkRow>(
sql, tenantScope.TenantId, driftId, ToDbValue(direction), offset, limit)
.ToListAsync(ct)
.ConfigureAwait(false);
return rows.Select(r => r.ToModel(direction)).ToList();
}
private static IEnumerable<object> EnumerateSinkRows(Guid driftId, ImmutableArray<DriftedSink> sinks, DriftDirection direction)
private static IEnumerable<SinkInsertParams> EnumerateSinkParams(
Guid driftId,
Guid tenantId,
ImmutableArray<DriftedSink> sinks,
DriftDirection direction)
{
foreach (var sink in sinks)
{
@@ -321,30 +290,35 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
? null
: JsonSerializer.Serialize(sink.AssociatedVulns, JsonOptions);
yield return new
{
sink.Id,
TenantId,
DriftId = driftId,
SinkNodeId = sink.SinkNodeId,
Symbol = sink.Symbol,
SinkCategory = ToDbValue(sink.SinkCategory),
Direction = ToDbValue(direction),
CauseKind = ToDbValue(sink.Cause.Kind),
CauseDescription = sink.Cause.Description,
CauseSymbol = sink.Cause.ChangedSymbol,
CauseFile = sink.Cause.ChangedFile,
CauseLine = sink.Cause.ChangedLine,
CodeChangeId = sink.Cause.CodeChangeId,
CompressedPath = pathJson,
AssociatedVulns = vulnsJson
};
yield return new SinkInsertParams(
Id: sink.Id,
TenantId: tenantId,
DriftId: driftId,
SinkNodeId: sink.SinkNodeId,
Symbol: sink.Symbol,
SinkCategory: ToDbValue(sink.SinkCategory),
Direction: ToDbValue(direction),
CauseKind: ToDbValue(sink.Cause.Kind),
CauseDescription: sink.Cause.Description,
CauseSymbol: sink.Cause.ChangedSymbol,
CauseFile: sink.Cause.ChangedFile,
CauseLine: sink.Cause.ChangedLine,
CodeChangeId: sink.Cause.CodeChangeId,
CompressedPath: pathJson,
AssociatedVulns: vulnsJson);
}
}
private sealed record SinkInsertParams(
Guid Id, Guid TenantId, Guid DriftId,
string SinkNodeId, string Symbol, string SinkCategory, string Direction,
string CauseKind, string CauseDescription, string? CauseSymbol, string? CauseFile,
int? CauseLine, Guid? CodeChangeId, string CompressedPath, string? AssociatedVulns);
private async Task<ReachabilityDriftResult> LoadResultAsync(
System.Data.IDbConnection connection,
NpgsqlConnection connection,
DriftHeaderRow header,
Guid tenantId,
CancellationToken ct)
{
var sinksSql = $"""
@@ -363,14 +337,16 @@ public sealed class PostgresReachabilityDriftResultRepository : IReachabilityDri
compressed_path,
associated_vulns
FROM {DriftedSinksTable}
WHERE tenant_id = @TenantId AND drift_result_id = @DriftId
WHERE tenant_id = @p0 AND drift_result_id = @p1
ORDER BY direction ASC, sink_node_id ASC
""";
var rows = (await connection.QueryAsync<DriftSinkRow>(new CommandDefinition(
sinksSql,
new { TenantId, DriftId = header.id },
cancellationToken: ct)).ConfigureAwait(false)).ToList();
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<DriftSinkRow>(
sinksSql, tenantId, header.id)
.ToListAsync(ct)
.ConfigureAwait(false);
var reachable = rows
.Where(r => string.Equals(r.direction, ToDbValue(DriftDirection.BecameReachable), StringComparison.Ordinal))

75
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresReachabilityResultRepository.cs
View File

@@ -1,5 +1,5 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Contracts;
using StellaOps.Scanner.Storage.Repositories;
@@ -9,9 +9,6 @@ namespace StellaOps.Scanner.Storage.Postgres;
public sealed class PostgresReachabilityResultRepository : IReachabilityResultRepository
{
private const string TenantContext = "00000000-0000-0000-0000-000000000001";
private static readonly Guid TenantId = Guid.Parse(TenantContext);
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = false
@@ -31,32 +28,18 @@ public sealed class PostgresReachabilityResultRepository : IReachabilityResultRe
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task StoreAsync(ReachabilityAnalysisResult result, CancellationToken ct = default)
public async Task StoreAsync(ReachabilityAnalysisResult result, CancellationToken ct = default, string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(result);
var trimmed = result.Trimmed();
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
INSERT INTO {ReachabilityResultsTable} (
tenant_id,
scan_id,
language,
graph_digest,
result_digest,
computed_at,
reachable_node_count,
reachable_sink_count,
result_json
tenant_id, scan_id, language, graph_digest, result_digest,
computed_at, reachable_node_count, reachable_sink_count, result_json
) VALUES (
@TenantId,
@ScanId,
@Language,
@GraphDigest,
@ResultDigest,
@ComputedAt,
@ReachableNodeCount,
@ReachableSinkCount,
@ResultJson::jsonb
@p0, @p1, @p2, @p3, @p4, @p5, @p6, @p7, @p8::jsonb
)
ON CONFLICT (tenant_id, scan_id, language, graph_digest, result_digest) DO UPDATE SET
computed_at = EXCLUDED.computed_at,
@@ -67,19 +50,19 @@ public sealed class PostgresReachabilityResultRepository : IReachabilityResultRe
var json = JsonSerializer.Serialize(trimmed, JsonOptions);
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
await connection.ExecuteAsync(new CommandDefinition(sql, new
{
TenantId = TenantId,
ScanId = trimmed.ScanId,
Language = trimmed.Language,
GraphDigest = trimmed.GraphDigest,
ResultDigest = trimmed.ResultDigest,
ComputedAt = trimmed.ComputedAt.UtcDateTime,
ReachableNodeCount = trimmed.ReachableNodeIds.Length,
ReachableSinkCount = trimmed.ReachableSinkIds.Length,
ResultJson = json
}, cancellationToken: ct)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
await dbContext.Database.ExecuteSqlRawAsync(
sql,
[
tenantScope.TenantId, trimmed.ScanId, trimmed.Language,
trimmed.GraphDigest, trimmed.ResultDigest,
trimmed.ComputedAt.UtcDateTime,
trimmed.ReachableNodeIds.Length, trimmed.ReachableSinkIds.Length,
json
],
ct).ConfigureAwait(false);
_logger.LogDebug(
"Stored reachability result scan={ScanId} lang={Language} sinks={Sinks}",
@@ -88,26 +71,27 @@ public sealed class PostgresReachabilityResultRepository : IReachabilityResultRe
trimmed.ReachableSinkIds.Length);
}
public async Task<ReachabilityAnalysisResult?> TryGetLatestAsync(string scanId, string language, CancellationToken ct = default)
public async Task<ReachabilityAnalysisResult?> TryGetLatestAsync(string scanId, string language, CancellationToken ct = default, string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
ArgumentException.ThrowIfNullOrWhiteSpace(language);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT result_json
FROM {ReachabilityResultsTable}
WHERE tenant_id = @TenantId AND scan_id = @ScanId AND language = @Language
WHERE tenant_id = @p0 AND scan_id = @p1 AND language = @p2
ORDER BY computed_at DESC
LIMIT 1
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var json = await connection.ExecuteScalarAsync<string?>(new CommandDefinition(sql, new
{
TenantId = TenantId,
ScanId = scanId,
Language = language
}, cancellationToken: ct)).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var json = await dbContext.Database.SqlQueryRaw<string>(
sql, tenantScope.TenantId, scanId, language)
.FirstOrDefaultAsync(ct)
.ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(json))
{
@@ -117,4 +101,3 @@ public sealed class PostgresReachabilityResultRepository : IReachabilityResultRe
return JsonSerializer.Deserialize<ReachabilityAnalysisResult>(json, JsonOptions);
}
}

162
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresRiskStateRepository.cs
View File

@@ -1,5 +1,5 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Scanner.SmartDiff.Detection;
@@ -13,9 +13,6 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// </summary>
public sealed class PostgresRiskStateRepository : IRiskStateRepository
{
private const string TenantContext = "00000000-0000-0000-0000-000000000001";
private static readonly Guid TenantId = Guid.Parse(TenantContext);
private readonly ScannerDataSource _dataSource;
private readonly ILogger<PostgresRiskStateRepository> _logger;
@@ -30,15 +27,16 @@ public sealed class PostgresRiskStateRepository : IRiskStateRepository
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task StoreSnapshotAsync(RiskStateSnapshot snapshot, CancellationToken ct = default)
public async Task StoreSnapshotAsync(RiskStateSnapshot snapshot, CancellationToken ct = default, string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(snapshot);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
await InsertSnapshotAsync(connection, snapshot, ct).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await InsertSnapshotAsync(connection, snapshot, tenantScope.TenantId, ct).ConfigureAwait(false);
}
public async Task StoreSnapshotsAsync(IReadOnlyList<RiskStateSnapshot> snapshots, CancellationToken ct = default)
public async Task StoreSnapshotsAsync(IReadOnlyList<RiskStateSnapshot> snapshots, CancellationToken ct = default, string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(snapshots);
@@ -47,14 +45,16 @@ public sealed class PostgresRiskStateRepository : IRiskStateRepository
return;
}
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false);
try
{
foreach (var snapshot in snapshots)
{
await InsertSnapshotAsync(connection, snapshot, ct, transaction).ConfigureAwait(false);
await InsertSnapshotAsync(connection, snapshot, tenantScope.TenantId, ct, transaction).ConfigureAwait(false);
}
await transaction.CommitAsync(ct).ConfigureAwait(false);
@@ -66,51 +66,58 @@ public sealed class PostgresRiskStateRepository : IRiskStateRepository
}
}
public async Task<RiskStateSnapshot?> GetLatestSnapshotAsync(FindingKey findingKey, CancellationToken ct = default)
public async Task<RiskStateSnapshot?> GetLatestSnapshotAsync(FindingKey findingKey, CancellationToken ct = default, string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(findingKey);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
vuln_id, purl, scan_id, captured_at,
reachable, lattice_state, vex_status::TEXT, in_affected_range,
kev, epss_score, policy_flags, policy_decision::TEXT, state_hash
reachable, lattice_state, vex_status::TEXT AS vex_status, in_affected_range,
kev, epss_score, policy_flags, policy_decision::TEXT AS policy_decision, state_hash
FROM {RiskStateSnapshotsTable}
WHERE tenant_id = @TenantId
AND vuln_id = @VulnId
AND purl = @Purl
WHERE tenant_id = @p0
AND vuln_id = @p1
AND purl = @p2
ORDER BY captured_at DESC
LIMIT 1
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var row = await connection.QuerySingleOrDefaultAsync<RiskStateRow>(sql, new
{
TenantId,
VulnId = findingKey.VulnId,
Purl = findingKey.ComponentPurl
});
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var row = await dbContext.Database.SqlQueryRaw<RiskStateRow>(
sql, tenantScope.TenantId, findingKey.VulnId, findingKey.ComponentPurl)
.FirstOrDefaultAsync(ct)
.ConfigureAwait(false);
return row?.ToSnapshot();
}
public async Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsForScanAsync(string scanId, CancellationToken ct = default)
public async Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsForScanAsync(string scanId, CancellationToken ct = default, string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
vuln_id, purl, scan_id, captured_at,
reachable, lattice_state, vex_status::TEXT, in_affected_range,
kev, epss_score, policy_flags, policy_decision::TEXT, state_hash
reachable, lattice_state, vex_status::TEXT AS vex_status, in_affected_range,
kev, epss_score, policy_flags, policy_decision::TEXT AS policy_decision, state_hash
FROM {RiskStateSnapshotsTable}
WHERE tenant_id = @TenantId
AND scan_id = @ScanId
WHERE tenant_id = @p0
AND scan_id = @p1
ORDER BY vuln_id, purl
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var rows = await connection.QueryAsync<RiskStateRow>(sql, new { TenantId, ScanId = scanId.Trim() });
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<RiskStateRow>(
sql, tenantScope.TenantId, scanId.Trim())
.ToListAsync(ct)
.ConfigureAwait(false);
return rows.Select(r => r.ToSnapshot()).ToList();
}
@@ -118,53 +125,60 @@ public sealed class PostgresRiskStateRepository : IRiskStateRepository
public async Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotHistoryAsync(
FindingKey findingKey,
int limit = 10,
CancellationToken ct = default)
CancellationToken ct = default,
string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(findingKey);
ArgumentOutOfRangeException.ThrowIfLessThan(limit, 1);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
vuln_id, purl, scan_id, captured_at,
reachable, lattice_state, vex_status::TEXT, in_affected_range,
kev, epss_score, policy_flags, policy_decision::TEXT, state_hash
reachable, lattice_state, vex_status::TEXT AS vex_status, in_affected_range,
kev, epss_score, policy_flags, policy_decision::TEXT AS policy_decision, state_hash
FROM {RiskStateSnapshotsTable}
WHERE tenant_id = @TenantId
AND vuln_id = @VulnId
AND purl = @Purl
WHERE tenant_id = @p0
AND vuln_id = @p1
AND purl = @p2
ORDER BY captured_at DESC
LIMIT @Limit
LIMIT @p3
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var rows = await connection.QueryAsync<RiskStateRow>(sql, new
{
TenantId,
VulnId = findingKey.VulnId,
Purl = findingKey.ComponentPurl,
Limit = limit
});
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<RiskStateRow>(
sql, tenantScope.TenantId, findingKey.VulnId, findingKey.ComponentPurl, limit)
.ToListAsync(ct)
.ConfigureAwait(false);
return rows.Select(r => r.ToSnapshot()).ToList();
}
public async Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsByHashAsync(string stateHash, CancellationToken ct = default)
public async Task<IReadOnlyList<RiskStateSnapshot>> GetSnapshotsByHashAsync(string stateHash, CancellationToken ct = default, string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(stateHash);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
vuln_id, purl, scan_id, captured_at,
reachable, lattice_state, vex_status::TEXT, in_affected_range,
kev, epss_score, policy_flags, policy_decision::TEXT, state_hash
reachable, lattice_state, vex_status::TEXT AS vex_status, in_affected_range,
kev, epss_score, policy_flags, policy_decision::TEXT AS policy_decision, state_hash
FROM {RiskStateSnapshotsTable}
WHERE tenant_id = @TenantId
AND state_hash = @StateHash
WHERE tenant_id = @p0
AND state_hash = @p1
ORDER BY captured_at DESC
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var rows = await connection.QueryAsync<RiskStateRow>(sql, new { TenantId, StateHash = stateHash.Trim() });
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<RiskStateRow>(
sql, tenantScope.TenantId, stateHash.Trim())
.ToListAsync(ct)
.ConfigureAwait(false);
return rows.Select(r => r.ToSnapshot()).ToList();
}
@@ -172,6 +186,7 @@ public sealed class PostgresRiskStateRepository : IRiskStateRepository
private async Task InsertSnapshotAsync(
NpgsqlConnection connection,
RiskStateSnapshot snapshot,
Guid tenantId,
CancellationToken ct,
NpgsqlTransaction? transaction = null)
{
@@ -183,9 +198,8 @@ public sealed class PostgresRiskStateRepository : IRiskStateRepository
reachable, lattice_state, vex_status, in_affected_range,
kev, epss_score, policy_flags, policy_decision, state_hash
) VALUES (
@TenantId, @VulnId, @Purl, @ScanId, @CapturedAt,
@Reachable, @LatticeState, @VexStatus::vex_status_type, @InAffectedRange,
@Kev, @EpssScore, @PolicyFlags, @PolicyDecision::policy_decision_type, @StateHash
$1, $2, $3, $4, $5, $6, $7, $8::vex_status_type, $9,
$10, $11, $12, $13::policy_decision_type, $14
)
ON CONFLICT (tenant_id, scan_id, vuln_id, purl) DO UPDATE SET
reachable = EXCLUDED.reachable,
@@ -199,27 +213,23 @@ public sealed class PostgresRiskStateRepository : IRiskStateRepository
state_hash = EXCLUDED.state_hash
""";
await connection.ExecuteAsync(new CommandDefinition(
sql,
new
{
TenantId,
VulnId = snapshot.FindingKey.VulnId,
Purl = snapshot.FindingKey.ComponentPurl,
ScanId = snapshot.ScanId,
CapturedAt = snapshot.CapturedAt,
Reachable = snapshot.Reachable,
LatticeState = snapshot.LatticeState,
VexStatus = snapshot.VexStatus.ToString().ToLowerInvariant(),
InAffectedRange = snapshot.InAffectedRange,
Kev = snapshot.Kev,
EpssScore = snapshot.EpssScore,
PolicyFlags = snapshot.PolicyFlags.ToArray(),
PolicyDecision = snapshot.PolicyDecision?.ToString().ToLowerInvariant(),
StateHash = snapshot.ComputeStateHash()
},
transaction: transaction,
cancellationToken: ct)).ConfigureAwait(false);
await using var cmd = new NpgsqlCommand(sql, connection, transaction);
cmd.Parameters.AddWithValue(tenantId);
cmd.Parameters.AddWithValue(snapshot.FindingKey.VulnId);
cmd.Parameters.AddWithValue(snapshot.FindingKey.ComponentPurl);
cmd.Parameters.AddWithValue(snapshot.ScanId);
cmd.Parameters.AddWithValue(snapshot.CapturedAt);
cmd.Parameters.AddWithValue(snapshot.Reachable.HasValue ? snapshot.Reachable.Value : DBNull.Value);
cmd.Parameters.AddWithValue(snapshot.LatticeState is null ? DBNull.Value : snapshot.LatticeState);
cmd.Parameters.AddWithValue(snapshot.VexStatus.ToString().ToLowerInvariant());
cmd.Parameters.AddWithValue(snapshot.InAffectedRange.HasValue ? snapshot.InAffectedRange.Value : DBNull.Value);
cmd.Parameters.AddWithValue(snapshot.Kev);
cmd.Parameters.AddWithValue(snapshot.EpssScore.HasValue ? snapshot.EpssScore.Value : DBNull.Value);
cmd.Parameters.AddWithValue(snapshot.PolicyFlags.ToArray());
cmd.Parameters.AddWithValue(snapshot.PolicyDecision?.ToString().ToLowerInvariant() ?? (object)DBNull.Value);
cmd.Parameters.AddWithValue(snapshot.ComputeStateHash());
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
/// <summary>

146
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresScanManifestRepository.cs
View File

@@ -1,4 +1,5 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using StellaOps.Scanner.Storage.EfCore.Models;
using StellaOps.Scanner.Storage.Entities;
using StellaOps.Scanner.Storage.Repositories;
@@ -6,12 +7,12 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// PostgreSQL implementation of scan manifest repository.
/// Converted from Dapper to EF Core; complex SQL kept as raw where needed.
/// </summary>
public sealed class PostgresScanManifestRepository : IScanManifestRepository
{
private readonly ScannerDataSource _dataSource;
private string SchemaName => _dataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private string TableName => $"{SchemaName}.scan_manifest";
public PostgresScanManifestRepository(ScannerDataSource dataSource)
{
@@ -20,112 +21,97 @@ public sealed class PostgresScanManifestRepository : IScanManifestRepository
public async Task<ScanManifestRow?> GetByHashAsync(string manifestHash, CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
manifest_id AS ManifestId,
scan_id AS ScanId,
manifest_hash AS ManifestHash,
sbom_hash AS SbomHash,
rules_hash AS RulesHash,
feed_hash AS FeedHash,
policy_hash AS PolicyHash,
scan_started_at AS ScanStartedAt,
scan_completed_at AS ScanCompletedAt,
manifest_content AS ManifestContent,
scanner_version AS ScannerVersion,
created_at AS CreatedAt
FROM {TableName}
WHERE manifest_hash = @ManifestHash
ORDER BY created_at DESC
LIMIT 1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.QueryFirstOrDefaultAsync<ScanManifestRow>(
new CommandDefinition(sql, new { ManifestHash = manifestHash }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entity = await dbContext.ScanManifests
.AsNoTracking()
.Where(e => e.ManifestHash == manifestHash)
.OrderByDescending(e => e.CreatedAt)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapToRow(entity);
}
public async Task<ScanManifestRow?> GetByScanIdAsync(Guid scanId, CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
manifest_id AS ManifestId,
scan_id AS ScanId,
manifest_hash AS ManifestHash,
sbom_hash AS SbomHash,
rules_hash AS RulesHash,
feed_hash AS FeedHash,
policy_hash AS PolicyHash,
scan_started_at AS ScanStartedAt,
scan_completed_at AS ScanCompletedAt,
manifest_content AS ManifestContent,
scanner_version AS ScannerVersion,
created_at AS CreatedAt
FROM {TableName}
WHERE scan_id = @ScanId
ORDER BY created_at DESC
LIMIT 1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.QuerySingleOrDefaultAsync<ScanManifestRow>(
new CommandDefinition(sql, new { ScanId = scanId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entity = await dbContext.ScanManifests
.AsNoTracking()
.Where(e => e.ScanId == scanId)
.OrderByDescending(e => e.CreatedAt)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapToRow(entity);
}
public async Task<ScanManifestRow> SaveAsync(ScanManifestRow manifest, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(manifest);
// Use raw SQL for INSERT RETURNING + jsonb cast which EF Core does not natively support.
var sql = $"""
INSERT INTO {TableName} (
scan_id,
manifest_hash,
sbom_hash,
rules_hash,
feed_hash,
policy_hash,
scan_started_at,
scan_completed_at,
manifest_content,
scanner_version
INSERT INTO {SchemaName}.scan_manifest (
scan_id, manifest_hash, sbom_hash, rules_hash, feed_hash,
policy_hash, scan_started_at, scan_completed_at, manifest_content, scanner_version
) VALUES (
@ScanId,
@ManifestHash,
@SbomHash,
@RulesHash,
@FeedHash,
@PolicyHash,
@ScanStartedAt,
@ScanCompletedAt,
@ManifestContent::jsonb,
@ScannerVersion
@p0, @p1, @p2, @p3, @p4, @p5, @p6, @p7, @p8::jsonb, @p9
)
RETURNING manifest_id AS ManifestId, created_at AS CreatedAt
RETURNING manifest_id, created_at
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var result = await connection.QuerySingleAsync<(Guid ManifestId, DateTimeOffset CreatedAt)>(
new CommandDefinition(sql, manifest, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var result = await dbContext.Database.SqlQueryRaw<ManifestInsertResult>(
sql,
manifest.ScanId, manifest.ManifestHash, manifest.SbomHash, manifest.RulesHash,
manifest.FeedHash, manifest.PolicyHash, manifest.ScanStartedAt,
(object?)manifest.ScanCompletedAt ?? DBNull.Value,
manifest.ManifestContent, manifest.ScannerVersion)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
manifest.ManifestId = result.ManifestId;
manifest.CreatedAt = result.CreatedAt;
manifest.ManifestId = result.manifest_id;
manifest.CreatedAt = result.created_at;
return manifest;
}
public async Task MarkCompletedAsync(Guid manifestId, DateTimeOffset completedAt, CancellationToken cancellationToken = default)
{
var sql = $"""
UPDATE {TableName}
SET scan_completed_at = @CompletedAt
WHERE manifest_id = @ManifestId
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await connection.ExecuteAsync(
new CommandDefinition(sql, new { ManifestId = manifestId, CompletedAt = completedAt }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
await dbContext.ScanManifests
.Where(e => e.ManifestId == manifestId)
.ExecuteUpdateAsync(s => s.SetProperty(e => e.ScanCompletedAt, completedAt), cancellationToken)
.ConfigureAwait(false);
}
private static ScanManifestRow MapToRow(ScanManifestEntity entity) => new()
{
ManifestId = entity.ManifestId,
ScanId = entity.ScanId,
ManifestHash = entity.ManifestHash,
SbomHash = entity.SbomHash,
RulesHash = entity.RulesHash,
FeedHash = entity.FeedHash,
PolicyHash = entity.PolicyHash,
ScanStartedAt = entity.ScanStartedAt,
ScanCompletedAt = entity.ScanCompletedAt,
ManifestContent = entity.ManifestContent,
ScannerVersion = entity.ScannerVersion,
CreatedAt = entity.CreatedAt
};
// Internal record for raw SQL result mapping.
private sealed record ManifestInsertResult
{
public Guid manifest_id { get; init; }
public DateTimeOffset created_at { get; init; }
}
}

491
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresSecretDetectionSettingsRepository.cs
View File

@@ -3,9 +3,11 @@
// Sprint: SPRINT_20260104_006_BE (Secret Detection Configuration API)
// Task: SDC-004 - Add persistence
// Description: PostgreSQL implementation for secret detection settings.
// Converted from Dapper to EF Core; jsonb casts and optimistic concurrency kept as raw SQL.
// -----------------------------------------------------------------------------
using Dapper;
using Microsoft.EntityFrameworkCore;
using StellaOps.Scanner.Storage.EfCore.Models;
using StellaOps.Scanner.Storage.Entities;
using StellaOps.Scanner.Storage.Repositories;
@@ -13,12 +15,12 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// PostgreSQL implementation of secret detection settings repository.
/// Converted from Dapper to EF Core; jsonb casts and optimistic concurrency kept as raw SQL.
/// </summary>
public sealed class PostgresSecretDetectionSettingsRepository : ISecretDetectionSettingsRepository
{
private readonly ScannerDataSource _dataSource;
private string SchemaName => _dataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private string TableName => $"{SchemaName}.secret_detection_settings";
public PostgresSecretDetectionSettingsRepository(ScannerDataSource dataSource)
{
@@ -29,32 +31,15 @@ public sealed class PostgresSecretDetectionSettingsRepository : ISecretDetection
Guid tenantId,
CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
settings_id AS SettingsId,
tenant_id AS TenantId,
enabled AS Enabled,
revelation_policy AS RevelationPolicy,
enabled_rule_categories AS EnabledRuleCategories,
disabled_rule_ids AS DisabledRuleIds,
alert_settings AS AlertSettings,
max_file_size_bytes AS MaxFileSizeBytes,
excluded_file_extensions AS ExcludedFileExtensions,
excluded_paths AS ExcludedPaths,
scan_binary_files AS ScanBinaryFiles,
require_signed_rule_bundles AS RequireSignedRuleBundles,
version AS Version,
updated_at AS UpdatedAt,
updated_by AS UpdatedBy,
created_at AS CreatedAt
FROM {TableName}
WHERE tenant_id = @TenantId
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.QuerySingleOrDefaultAsync<SecretDetectionSettingsRow>(
new CommandDefinition(sql, new { TenantId = tenantId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var entity = await dbContext.SecretDetectionSettings
.AsNoTracking()
.FirstOrDefaultAsync(e => e.TenantId == tenantId, cancellationToken)
.ConfigureAwait(false);
return entity is null ? null : MapToRow(entity);
}
public async Task<SecretDetectionSettingsRow> CreateAsync(
@@ -63,46 +48,42 @@ public sealed class PostgresSecretDetectionSettingsRepository : ISecretDetection
{
ArgumentNullException.ThrowIfNull(settings);
// Keep raw SQL for INSERT RETURNING + jsonb casts.
var sql = $"""
INSERT INTO {TableName} (
tenant_id,
enabled,
revelation_policy,
enabled_rule_categories,
disabled_rule_ids,
alert_settings,
max_file_size_bytes,
excluded_file_extensions,
excluded_paths,
scan_binary_files,
require_signed_rule_bundles,
updated_by
INSERT INTO {SchemaName}.secret_detection_settings (
tenant_id, enabled, revelation_policy,
enabled_rule_categories, disabled_rule_ids,
alert_settings, max_file_size_bytes,
excluded_file_extensions, excluded_paths,
scan_binary_files, require_signed_rule_bundles, updated_by
) VALUES (
@TenantId,
@Enabled,
@RevelationPolicy::jsonb,
@EnabledRuleCategories,
@DisabledRuleIds,
@AlertSettings::jsonb,
@MaxFileSizeBytes,
@ExcludedFileExtensions,
@ExcludedPaths,
@ScanBinaryFiles,
@RequireSignedRuleBundles,
@UpdatedBy
@p0, @p1, @p2::jsonb, @p3, @p4, @p5::jsonb, @p6, @p7, @p8, @p9, @p10, @p11
)
RETURNING settings_id AS SettingsId, version AS Version, created_at AS CreatedAt, updated_at AS UpdatedAt
RETURNING settings_id, version, created_at, updated_at
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var result = await connection.QuerySingleAsync<(Guid SettingsId, int Version, DateTimeOffset CreatedAt, DateTimeOffset UpdatedAt)>(
new CommandDefinition(sql, settings, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var result = await dbContext.Database.SqlQueryRaw<SettingsInsertResult>(
sql,
settings.TenantId, settings.Enabled,
(object?)settings.RevelationPolicy ?? DBNull.Value,
(object?)settings.EnabledRuleCategories ?? DBNull.Value,
(object?)settings.DisabledRuleIds ?? DBNull.Value,
(object?)settings.AlertSettings ?? DBNull.Value,
settings.MaxFileSizeBytes,
(object?)settings.ExcludedFileExtensions ?? DBNull.Value,
(object?)settings.ExcludedPaths ?? DBNull.Value,
settings.ScanBinaryFiles, settings.RequireSignedRuleBundles,
(object?)settings.UpdatedBy ?? DBNull.Value)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
settings.SettingsId = result.SettingsId;
settings.Version = result.Version;
settings.CreatedAt = result.CreatedAt;
settings.UpdatedAt = result.UpdatedAt;
settings.SettingsId = result.settings_id;
settings.Version = result.version;
settings.CreatedAt = result.created_at;
settings.UpdatedAt = result.updated_at;
return settings;
}
@@ -113,43 +94,45 @@ public sealed class PostgresSecretDetectionSettingsRepository : ISecretDetection
{
ArgumentNullException.ThrowIfNull(settings);
// Keep raw SQL for optimistic concurrency + jsonb casts.
var sql = $"""
UPDATE {TableName}
UPDATE {SchemaName}.secret_detection_settings
SET
enabled = @Enabled,
revelation_policy = @RevelationPolicy::jsonb,
enabled_rule_categories = @EnabledRuleCategories,
disabled_rule_ids = @DisabledRuleIds,
alert_settings = @AlertSettings::jsonb,
max_file_size_bytes = @MaxFileSizeBytes,
excluded_file_extensions = @ExcludedFileExtensions,
excluded_paths = @ExcludedPaths,
scan_binary_files = @ScanBinaryFiles,
require_signed_rule_bundles = @RequireSignedRuleBundles,
enabled = @p0,
revelation_policy = @p1::jsonb,
enabled_rule_categories = @p2,
disabled_rule_ids = @p3,
alert_settings = @p4::jsonb,
max_file_size_bytes = @p5,
excluded_file_extensions = @p6,
excluded_paths = @p7,
scan_binary_files = @p8,
require_signed_rule_bundles = @p9,
version = version + 1,
updated_at = NOW(),
updated_by = @UpdatedBy
WHERE settings_id = @SettingsId AND version = @ExpectedVersion
updated_by = @p10
WHERE settings_id = @p11 AND version = @p12
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rowsAffected = await connection.ExecuteAsync(
new CommandDefinition(sql, new
{
settings.SettingsId,
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rowsAffected = await dbContext.Database.ExecuteSqlRawAsync(
sql,
[
settings.Enabled,
settings.RevelationPolicy,
settings.EnabledRuleCategories,
settings.DisabledRuleIds,
settings.AlertSettings,
(object?)settings.RevelationPolicy ?? DBNull.Value,
(object?)settings.EnabledRuleCategories ?? DBNull.Value,
(object?)settings.DisabledRuleIds ?? DBNull.Value,
(object?)settings.AlertSettings ?? DBNull.Value,
settings.MaxFileSizeBytes,
settings.ExcludedFileExtensions,
settings.ExcludedPaths,
settings.ScanBinaryFiles,
settings.RequireSignedRuleBundles,
settings.UpdatedBy,
ExpectedVersion = expectedVersion
}, cancellationToken: cancellationToken))
(object?)settings.ExcludedFileExtensions ?? DBNull.Value,
(object?)settings.ExcludedPaths ?? DBNull.Value,
settings.ScanBinaryFiles, settings.RequireSignedRuleBundles,
(object?)settings.UpdatedBy ?? DBNull.Value,
settings.SettingsId, expectedVersion
],
cancellationToken)
.ConfigureAwait(false);
return rowsAffected > 0;
@@ -157,24 +140,50 @@ public sealed class PostgresSecretDetectionSettingsRepository : ISecretDetection
public async Task<IReadOnlyList<Guid>> GetEnabledTenantsAsync(CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT tenant_id
FROM {TableName}
WHERE enabled = TRUE
ORDER BY tenant_id
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var result = await connection.QueryAsync<Guid>(
new CommandDefinition(sql, cancellationToken: cancellationToken))
.ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return result.ToList();
return await dbContext.SecretDetectionSettings
.AsNoTracking()
.Where(e => e.Enabled)
.OrderBy(e => e.TenantId)
.Select(e => e.TenantId)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
private static SecretDetectionSettingsRow MapToRow(SecretDetectionSettingsEntity e) => new()
{
SettingsId = e.SettingsId,
TenantId = e.TenantId,
Enabled = e.Enabled,
RevelationPolicy = e.RevelationPolicy ?? string.Empty,
EnabledRuleCategories = e.EnabledRuleCategories ?? [],
DisabledRuleIds = e.DisabledRuleIds ?? [],
AlertSettings = e.AlertSettings ?? string.Empty,
MaxFileSizeBytes = e.MaxFileSizeBytes,
ExcludedFileExtensions = e.ExcludedFileExtensions ?? [],
ExcludedPaths = e.ExcludedPaths ?? [],
ScanBinaryFiles = e.ScanBinaryFiles,
RequireSignedRuleBundles = e.RequireSignedRuleBundles,
Version = e.Version,
UpdatedAt = e.UpdatedAt,
UpdatedBy = e.UpdatedBy ?? string.Empty,
CreatedAt = e.CreatedAt
};
private sealed record SettingsInsertResult
{
public Guid settings_id { get; init; }
public int version { get; init; }
public DateTimeOffset created_at { get; init; }
public DateTimeOffset updated_at { get; init; }
}
}
/// <summary>
/// PostgreSQL implementation of secret exception pattern repository.
/// Converted from Dapper to EF Core raw SQL (tables not modeled in DbContext).
/// </summary>
public sealed class PostgresSecretExceptionPatternRepository : ISecretExceptionPatternRepository
{
@@ -194,66 +203,69 @@ public sealed class PostgresSecretExceptionPatternRepository : ISecretExceptionP
{
var sql = $"""
SELECT
exception_id AS ExceptionId,
tenant_id AS TenantId,
name AS Name,
description AS Description,
value_pattern AS ValuePattern,
applicable_rule_ids AS ApplicableRuleIds,
file_path_glob AS FilePathGlob,
justification AS Justification,
expires_at AS ExpiresAt,
is_active AS IsActive,
match_count AS MatchCount,
last_matched_at AS LastMatchedAt,
created_at AS CreatedAt,
created_by AS CreatedBy,
updated_at AS UpdatedAt,
updated_by AS UpdatedBy
exception_id AS "ExceptionId",
tenant_id AS "TenantId",
name AS "Name",
description AS "Description",
value_pattern AS "ValuePattern",
applicable_rule_ids AS "ApplicableRuleIds",
file_path_glob AS "FilePathGlob",
justification AS "Justification",
expires_at AS "ExpiresAt",
is_active AS "IsActive",
match_count AS "MatchCount",
last_matched_at AS "LastMatchedAt",
created_at AS "CreatedAt",
created_by AS "CreatedBy",
updated_at AS "UpdatedAt",
updated_by AS "UpdatedBy"
FROM {PatternTableName}
WHERE tenant_id = @TenantId
WHERE tenant_id = @p0
AND is_active = TRUE
AND (expires_at IS NULL OR expires_at > NOW())
ORDER BY name
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var result = await connection.QueryAsync<SecretExceptionPatternRow>(
new CommandDefinition(sql, new { TenantId = tenantId }, cancellationToken: cancellationToken))
.ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return result.ToList();
return await dbContext.Database.SqlQueryRaw<SecretExceptionPatternRow>(sql, tenantId)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
public async Task<SecretExceptionPatternRow?> GetByIdAsync(
Guid tenantId,
Guid exceptionId,
CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT
exception_id AS ExceptionId,
tenant_id AS TenantId,
name AS Name,
description AS Description,
value_pattern AS ValuePattern,
applicable_rule_ids AS ApplicableRuleIds,
file_path_glob AS FilePathGlob,
justification AS Justification,
expires_at AS ExpiresAt,
is_active AS IsActive,
match_count AS MatchCount,
last_matched_at AS LastMatchedAt,
created_at AS CreatedAt,
created_by AS CreatedBy,
updated_at AS UpdatedAt,
updated_by AS UpdatedBy
exception_id AS "ExceptionId",
tenant_id AS "TenantId",
name AS "Name",
description AS "Description",
value_pattern AS "ValuePattern",
applicable_rule_ids AS "ApplicableRuleIds",
file_path_glob AS "FilePathGlob",
justification AS "Justification",
expires_at AS "ExpiresAt",
is_active AS "IsActive",
match_count AS "MatchCount",
last_matched_at AS "LastMatchedAt",
created_at AS "CreatedAt",
created_by AS "CreatedBy",
updated_at AS "UpdatedAt",
updated_by AS "UpdatedBy"
FROM {PatternTableName}
WHERE exception_id = @ExceptionId
WHERE tenant_id = @p0 AND exception_id = @p1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
return await connection.QuerySingleOrDefaultAsync<SecretExceptionPatternRow>(
new CommandDefinition(sql, new { ExceptionId = exceptionId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return await dbContext.Database.SqlQueryRaw<SecretExceptionPatternRow>(sql, tenantId, exceptionId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
}
@@ -265,42 +277,39 @@ public sealed class PostgresSecretExceptionPatternRepository : ISecretExceptionP
var sql = $"""
INSERT INTO {PatternTableName} (
tenant_id,
name,
description,
value_pattern,
applicable_rule_ids,
file_path_glob,
justification,
expires_at,
is_active,
created_by
tenant_id, name, description, value_pattern,
applicable_rule_ids, file_path_glob, justification,
expires_at, is_active, created_by
) VALUES (
@TenantId,
@Name,
@Description,
@ValuePattern,
@ApplicableRuleIds,
@FilePathGlob,
@Justification,
@ExpiresAt,
@IsActive,
@CreatedBy
@p0, @p1, @p2, @p3, @p4, @p5, @p6, @p7, @p8, @p9
)
RETURNING exception_id AS ExceptionId, created_at AS CreatedAt
RETURNING exception_id, created_at
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var result = await connection.QuerySingleAsync<(Guid ExceptionId, DateTimeOffset CreatedAt)>(
new CommandDefinition(sql, pattern, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var result = await dbContext.Database.SqlQueryRaw<ExceptionPatternInsertResult>(
sql,
pattern.TenantId, pattern.Name,
(object?)pattern.Description ?? DBNull.Value,
pattern.ValuePattern,
(object?)pattern.ApplicableRuleIds ?? DBNull.Value,
(object?)pattern.FilePathGlob ?? DBNull.Value,
(object?)pattern.Justification ?? DBNull.Value,
(object?)pattern.ExpiresAt ?? DBNull.Value,
pattern.IsActive,
(object?)pattern.CreatedBy ?? DBNull.Value)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
pattern.ExceptionId = result.ExceptionId;
pattern.CreatedAt = result.CreatedAt;
pattern.ExceptionId = result.exception_id;
pattern.CreatedAt = result.created_at;
return pattern;
}
public async Task<bool> UpdateAsync(
Guid tenantId,
SecretExceptionPatternRow pattern,
CancellationToken cancellationToken = default)
{
@@ -309,39 +318,50 @@ public sealed class PostgresSecretExceptionPatternRepository : ISecretExceptionP
var sql = $"""
UPDATE {PatternTableName}
SET
name = @Name,
description = @Description,
value_pattern = @ValuePattern,
applicable_rule_ids = @ApplicableRuleIds,
file_path_glob = @FilePathGlob,
justification = @Justification,
expires_at = @ExpiresAt,
is_active = @IsActive,
updated_at = NOW(),
updated_by = @UpdatedBy
WHERE exception_id = @ExceptionId
name = @p0, description = @p1, value_pattern = @p2,
applicable_rule_ids = @p3, file_path_glob = @p4,
justification = @p5, expires_at = @p6, is_active = @p7,
updated_at = NOW(), updated_by = @p8
WHERE tenant_id = @p9 AND exception_id = @p10
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rowsAffected = await connection.ExecuteAsync(
new CommandDefinition(sql, pattern, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rowsAffected = await dbContext.Database.ExecuteSqlRawAsync(
sql,
[
pattern.Name,
(object?)pattern.Description ?? DBNull.Value,
pattern.ValuePattern,
(object?)pattern.ApplicableRuleIds ?? DBNull.Value,
(object?)pattern.FilePathGlob ?? DBNull.Value,
(object?)pattern.Justification ?? DBNull.Value,
(object?)pattern.ExpiresAt ?? DBNull.Value,
pattern.IsActive,
(object?)pattern.UpdatedBy ?? DBNull.Value,
tenantId, pattern.ExceptionId
],
cancellationToken)
.ConfigureAwait(false);
return rowsAffected > 0;
}
public async Task<bool> DeleteAsync(
Guid tenantId,
Guid exceptionId,
CancellationToken cancellationToken = default)
{
var sql = $"""
DELETE FROM {PatternTableName}
WHERE exception_id = @ExceptionId
WHERE tenant_id = @p0 AND exception_id = @p1
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var rowsAffected = await connection.ExecuteAsync(
new CommandDefinition(sql, new { ExceptionId = exceptionId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rowsAffected = await dbContext.Database.ExecuteSqlRawAsync(sql, [tenantId, exceptionId], cancellationToken)
.ConfigureAwait(false);
return rowsAffected > 0;
@@ -353,33 +373,33 @@ public sealed class PostgresSecretExceptionPatternRepository : ISecretExceptionP
{
var sql = $"""
SELECT
exception_id AS ExceptionId,
tenant_id AS TenantId,
name AS Name,
description AS Description,
value_pattern AS ValuePattern,
applicable_rule_ids AS ApplicableRuleIds,
file_path_glob AS FilePathGlob,
justification AS Justification,
expires_at AS ExpiresAt,
is_active AS IsActive,
match_count AS MatchCount,
last_matched_at AS LastMatchedAt,
created_at AS CreatedAt,
created_by AS CreatedBy,
updated_at AS UpdatedAt,
updated_by AS UpdatedBy
exception_id AS "ExceptionId",
tenant_id AS "TenantId",
name AS "Name",
description AS "Description",
value_pattern AS "ValuePattern",
applicable_rule_ids AS "ApplicableRuleIds",
file_path_glob AS "FilePathGlob",
justification AS "Justification",
expires_at AS "ExpiresAt",
is_active AS "IsActive",
match_count AS "MatchCount",
last_matched_at AS "LastMatchedAt",
created_at AS "CreatedAt",
created_by AS "CreatedBy",
updated_at AS "UpdatedAt",
updated_by AS "UpdatedBy"
FROM {PatternTableName}
WHERE tenant_id = @TenantId
WHERE tenant_id = @p0
ORDER BY name
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var result = await connection.QueryAsync<SecretExceptionPatternRow>(
new CommandDefinition(sql, new { TenantId = tenantId }, cancellationToken: cancellationToken))
.ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return result.ToList();
return await dbContext.Database.SqlQueryRaw<SecretExceptionPatternRow>(sql, tenantId)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
public async Task RecordMatchAsync(
@@ -392,23 +412,20 @@ public sealed class PostgresSecretExceptionPatternRepository : ISecretExceptionP
{
var sql = $"""
INSERT INTO {MatchLogTableName} (
tenant_id,
exception_id,
scan_id,
file_path,
rule_id
) VALUES (
@TenantId,
@ExceptionId,
@ScanId,
@FilePath,
@RuleId
)
tenant_id, exception_id, scan_id, file_path, rule_id
) VALUES (@p0, @p1, @p2, @p3, @p4)
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
await connection.ExecuteAsync(
new CommandDefinition(sql, new { TenantId = tenantId, ExceptionId = exceptionId, ScanId = scanId, FilePath = filePath, RuleId = ruleId }, cancellationToken: cancellationToken))
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
await dbContext.Database.ExecuteSqlRawAsync(
sql,
[tenantId, exceptionId,
(object?)scanId ?? DBNull.Value,
(object?)filePath ?? DBNull.Value,
(object?)ruleId ?? DBNull.Value],
cancellationToken)
.ConfigureAwait(false);
}
@@ -418,34 +435,40 @@ public sealed class PostgresSecretExceptionPatternRepository : ISecretExceptionP
{
var sql = $"""
SELECT
exception_id AS ExceptionId,
tenant_id AS TenantId,
name AS Name,
description AS Description,
value_pattern AS ValuePattern,
applicable_rule_ids AS ApplicableRuleIds,
file_path_glob AS FilePathGlob,
justification AS Justification,
expires_at AS ExpiresAt,
is_active AS IsActive,
match_count AS MatchCount,
last_matched_at AS LastMatchedAt,
created_at AS CreatedAt,
created_by AS CreatedBy,
updated_at AS UpdatedAt,
updated_by AS UpdatedBy
exception_id AS "ExceptionId",
tenant_id AS "TenantId",
name AS "Name",
description AS "Description",
value_pattern AS "ValuePattern",
applicable_rule_ids AS "ApplicableRuleIds",
file_path_glob AS "FilePathGlob",
justification AS "Justification",
expires_at AS "ExpiresAt",
is_active AS "IsActive",
match_count AS "MatchCount",
last_matched_at AS "LastMatchedAt",
created_at AS "CreatedAt",
created_by AS "CreatedBy",
updated_at AS "UpdatedAt",
updated_by AS "UpdatedBy"
FROM {PatternTableName}
WHERE expires_at IS NOT NULL
AND expires_at <= @AsOf
AND expires_at <= @p0
AND is_active = TRUE
ORDER BY expires_at
""";
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken).ConfigureAwait(false);
var result = await connection.QueryAsync<SecretExceptionPatternRow>(
new CommandDefinition(sql, new { AsOf = asOf }, cancellationToken: cancellationToken))
.ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
return result.ToList();
return await dbContext.Database.SqlQueryRaw<SecretExceptionPatternRow>(sql, asOf)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
private sealed record ExceptionPatternInsertResult
{
public Guid exception_id { get; init; }
public DateTimeOffset created_at { get; init; }
}
}

137
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/PostgresVexCandidateStore.cs
View File

@@ -1,5 +1,5 @@
using Dapper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Scanner.SmartDiff.Detection;
@@ -14,9 +14,6 @@ namespace StellaOps.Scanner.Storage.Postgres;
/// </summary>
public sealed class PostgresVexCandidateStore : IVexCandidateStore
{
private const string TenantContext = "00000000-0000-0000-0000-000000000001";
private static readonly Guid TenantId = Guid.Parse(TenantContext);
private readonly ScannerDataSource _dataSource;
private readonly ILogger<PostgresVexCandidateStore> _logger;
@@ -36,21 +33,22 @@ public sealed class PostgresVexCandidateStore : IVexCandidateStore
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task StoreCandidatesAsync(IReadOnlyList<VexCandidate> candidates, CancellationToken ct = default)
public async Task StoreCandidatesAsync(IReadOnlyList<VexCandidate> candidates, CancellationToken ct = default, string? tenantId = null)
{
ArgumentNullException.ThrowIfNull(candidates);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
if (candidates.Count == 0)
return;
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var transaction = await connection.BeginTransactionAsync(ct).ConfigureAwait(false);
try
{
foreach (var candidate in candidates)
{
await InsertCandidateAsync(connection, candidate, ct, transaction).ConfigureAwait(false);
await InsertCandidateAsync(connection, candidate, tenantScope.TenantId, ct, transaction).ConfigureAwait(false);
}
await transaction.CommitAsync(ct).ConfigureAwait(false);
@@ -64,75 +62,92 @@ public sealed class PostgresVexCandidateStore : IVexCandidateStore
}
}
public async Task<IReadOnlyList<VexCandidate>> GetCandidatesAsync(string imageDigest, CancellationToken ct = default)
public async Task<IReadOnlyList<VexCandidate>> GetCandidatesAsync(string imageDigest, CancellationToken ct = default, string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
SELECT
candidate_id, vuln_id, purl, image_digest,
suggested_status::TEXT, justification::TEXT, rationale,
suggested_status::TEXT AS suggested_status, justification::TEXT AS justification, rationale,
evidence_links, confidence, generated_at, expires_at,
requires_review, review_action::TEXT, reviewed_by, reviewed_at, review_comment
requires_review, review_action::TEXT AS review_action, reviewed_by, reviewed_at, review_comment
FROM {VexCandidatesTable}
WHERE tenant_id = @TenantId
AND image_digest = @ImageDigest
WHERE tenant_id = @p0
AND image_digest = @p1
ORDER BY confidence DESC
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var rows = await connection.QueryAsync<VexCandidateRow>(sql, new { TenantId, ImageDigest = imageDigest.Trim() });
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var rows = await dbContext.Database.SqlQueryRaw<VexCandidateRow>(
sql, tenantScope.TenantId, imageDigest.Trim())
.ToListAsync(ct)
.ConfigureAwait(false);
return rows.Select(r => r.ToCandidate()).ToList();
}
public async Task<VexCandidate?> GetCandidateAsync(string candidateId, CancellationToken ct = default)
public async Task<VexCandidate?> GetCandidateAsync(string candidateId, CancellationToken ct = default, string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(candidateId);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
SELECT
SELECT
candidate_id, vuln_id, purl, image_digest,
suggested_status::TEXT, justification::TEXT, rationale,
suggested_status::TEXT AS suggested_status, justification::TEXT AS justification, rationale,
evidence_links, confidence, generated_at, expires_at,
requires_review, review_action::TEXT, reviewed_by, reviewed_at, review_comment
requires_review, review_action::TEXT AS review_action, reviewed_by, reviewed_at, review_comment
FROM {VexCandidatesTable}
WHERE tenant_id = @TenantId
AND candidate_id = @CandidateId
WHERE tenant_id = @p0
AND candidate_id = @p1
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var row = await connection.QuerySingleOrDefaultAsync<VexCandidateRow>(sql, new { TenantId, CandidateId = candidateId.Trim() });
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var row = await dbContext.Database.SqlQueryRaw<VexCandidateRow>(
sql, tenantScope.TenantId, candidateId.Trim())
.FirstOrDefaultAsync(ct)
.ConfigureAwait(false);
return row?.ToCandidate();
}
public async Task<bool> ReviewCandidateAsync(string candidateId, VexCandidateReview review, CancellationToken ct = default)
public async Task<bool> ReviewCandidateAsync(string candidateId, VexCandidateReview review, CancellationToken ct = default, string? tenantId = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(candidateId);
ArgumentNullException.ThrowIfNull(review);
var tenantScope = ScannerTenantScope.Resolve(tenantId);
var sql = $"""
UPDATE {VexCandidatesTable} SET
requires_review = FALSE,
review_action = @ReviewAction::vex_review_action,
reviewed_by = @ReviewedBy,
reviewed_at = @ReviewedAt,
review_comment = @ReviewComment
WHERE tenant_id = @TenantId
AND candidate_id = @CandidateId
review_action = @p0::vex_review_action,
reviewed_by = @p1,
reviewed_at = @p2,
review_comment = @p3
WHERE tenant_id = @p4
AND candidate_id = @p5
""";
await using var connection = await _dataSource.OpenConnectionAsync(TenantContext, ct).ConfigureAwait(false);
var affected = await connection.ExecuteAsync(sql, new
{
TenantId,
CandidateId = candidateId.Trim(),
ReviewAction = review.Action.ToString().ToLowerInvariant(),
ReviewedBy = review.Reviewer,
ReviewedAt = review.ReviewedAt,
ReviewComment = review.Comment
});
await using var connection = await _dataSource.OpenConnectionAsync(tenantScope.TenantContext, ct).ConfigureAwait(false);
await using var dbContext = ScannerDbContextFactory.Create(connection, _dataSource.CommandTimeoutSeconds, SchemaName);
var affected = await dbContext.Database.ExecuteSqlRawAsync(
sql,
[
review.Action.ToString().ToLowerInvariant(),
review.Reviewer,
review.ReviewedAt,
(object?)review.Comment ?? DBNull.Value,
tenantScope.TenantId,
candidateId.Trim()
],
ct).ConfigureAwait(false);
if (affected > 0)
{
@@ -146,6 +161,7 @@ public sealed class PostgresVexCandidateStore : IVexCandidateStore
private async Task InsertCandidateAsync(
NpgsqlConnection connection,
VexCandidate candidate,
Guid tenantId,
CancellationToken ct,
NpgsqlTransaction? transaction = null)
{
@@ -158,9 +174,9 @@ public sealed class PostgresVexCandidateStore : IVexCandidateStore
suggested_status, justification, rationale,
evidence_links, confidence, generated_at, expires_at, requires_review
) VALUES (
@TenantId, @CandidateId, @VulnId, @Purl, @ImageDigest,
@SuggestedStatus::vex_status_type, @Justification::vex_justification, @Rationale,
@EvidenceLinks::jsonb, @Confidence, @GeneratedAt, @ExpiresAt, @RequiresReview
$1, $2, $3, $4, $5,
$6::vex_status_type, $7::vex_justification, $8,
$9::jsonb, $10, $11, $12, $13
)
ON CONFLICT (candidate_id) DO UPDATE SET
suggested_status = EXCLUDED.suggested_status,
@@ -171,25 +187,24 @@ public sealed class PostgresVexCandidateStore : IVexCandidateStore
expires_at = EXCLUDED.expires_at
""";
var tenantId = TenantId;
var evidenceLinksJson = JsonSerializer.Serialize(candidate.EvidenceLinks, JsonOptions);
await connection.ExecuteAsync(new CommandDefinition(sql, new
{
TenantId = tenantId,
CandidateId = candidate.CandidateId,
VulnId = candidate.FindingKey.VulnId,
Purl = candidate.FindingKey.ComponentPurl,
ImageDigest = candidate.ImageDigest,
SuggestedStatus = MapVexStatus(candidate.SuggestedStatus),
Justification = MapJustification(candidate.Justification),
Rationale = candidate.Rationale,
EvidenceLinks = evidenceLinksJson,
Confidence = candidate.Confidence,
GeneratedAt = candidate.GeneratedAt,
ExpiresAt = candidate.ExpiresAt,
RequiresReview = candidate.RequiresReview
}, transaction: transaction, cancellationToken: ct));
await using var cmd = new NpgsqlCommand(sql, connection, transaction);
cmd.Parameters.AddWithValue(tenantId);
cmd.Parameters.AddWithValue(candidate.CandidateId);
cmd.Parameters.AddWithValue(candidate.FindingKey.VulnId);
cmd.Parameters.AddWithValue(candidate.FindingKey.ComponentPurl);
cmd.Parameters.AddWithValue(candidate.ImageDigest);
cmd.Parameters.AddWithValue(MapVexStatus(candidate.SuggestedStatus));
cmd.Parameters.AddWithValue(MapJustification(candidate.Justification));
cmd.Parameters.AddWithValue(candidate.Rationale);
cmd.Parameters.AddWithValue(evidenceLinksJson);
cmd.Parameters.AddWithValue(candidate.Confidence);
cmd.Parameters.AddWithValue(candidate.GeneratedAt);
cmd.Parameters.AddWithValue(candidate.ExpiresAt);
cmd.Parameters.AddWithValue(candidate.RequiresReview);
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
private static string MapVexStatus(VexStatusType status)
@@ -218,7 +233,7 @@ public sealed class PostgresVexCandidateStore : IVexCandidateStore
}
/// <summary>
/// Row mapping class for Dapper.
/// Row mapping class for EF Core SqlQueryRaw.
/// </summary>
private sealed class VexCandidateRow
{

34
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/ScannerDbContextFactory.cs Normal file
View File

@@ -0,0 +1,34 @@
using Microsoft.EntityFrameworkCore;
using Npgsql;
using StellaOps.Scanner.Storage.EfCore.CompiledModels;
using StellaOps.Scanner.Storage.EfCore.Context;
namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// Runtime factory for creating <see cref="ScannerDbContext"/> instances.
/// Uses compiled model for default schema to avoid runtime model-building overhead.
/// </summary>
internal static class ScannerDbContextFactory
{
public static ScannerDbContext Create(NpgsqlConnection connection, int commandTimeoutSeconds, string schemaName)
{
var normalizedSchema = string.IsNullOrWhiteSpace(schemaName)
? ScannerStorageDefaults.DefaultSchemaName
: schemaName.Trim();
var optionsBuilder = new DbContextOptionsBuilder<ScannerDbContext>()
.UseNpgsql(connection, npgsql => npgsql.CommandTimeout(commandTimeoutSeconds));
if (string.Equals(normalizedSchema, ScannerStorageDefaults.DefaultSchemaName, StringComparison.Ordinal))
{
// Use the static compiled model when schema matches the default for faster context startup.
if (ScannerDbContextModel.Instance.GetEntityTypes().Any())
{
optionsBuilder.UseModel(ScannerDbContextModel.Instance);
}
}
return new ScannerDbContext(optionsBuilder.Options, normalizedSchema);
}
}

27
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/ScannerTenantScope.cs Normal file
View File

@@ -0,0 +1,27 @@
using StellaOps.Scanner.Core.Utility;
using System.Text;
namespace StellaOps.Scanner.Storage.Postgres;
internal static class ScannerTenantScope
{
private const string DefaultTenant = "default";
private static readonly Guid TenantNamespace = new("ac8f2b54-72ea-43fa-9c3b-6a87ebd2d48a");
public static (string TenantContext, Guid TenantId) Resolve(string? tenantId)
{
var normalizedTenant = string.IsNullOrWhiteSpace(tenantId)
? DefaultTenant
: tenantId.Trim().ToLowerInvariant();
if (Guid.TryParse(normalizedTenant, out var parsed))
{
return (parsed.ToString("D"), parsed);
}
var deterministic = ScannerIdentifiers.CreateDeterministicGuid(
TenantNamespace,
Encoding.UTF8.GetBytes(normalizedTenant));
return (deterministic.ToString("D"), deterministic);
}
}

5
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/ICallGraphSnapshotRepository.cs
View File

@@ -4,8 +4,7 @@ namespace StellaOps.Scanner.Storage.Repositories;
public interface ICallGraphSnapshotRepository
{
Task StoreAsync(CallGraphSnapshot snapshot, CancellationToken ct = default);
Task StoreAsync(CallGraphSnapshot snapshot, CancellationToken ct = default, string? tenantId = null);
Task<CallGraphSnapshot?> TryGetLatestAsync(string scanId, string language, CancellationToken ct = default);
Task<CallGraphSnapshot?> TryGetLatestAsync(string scanId, string language, CancellationToken ct = default, string? tenantId = null);
}

3
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/ICodeChangeRepository.cs
View File

@@ -4,6 +4,5 @@ namespace StellaOps.Scanner.Storage.Repositories;
public interface ICodeChangeRepository
{
Task StoreAsync(IReadOnlyList<CodeChangeFact> changes, CancellationToken ct = default);
Task StoreAsync(IReadOnlyList<CodeChangeFact> changes, CancellationToken ct = default, string? tenantId = null);
}

11
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/IReachabilityDriftResultRepository.cs
View File

@@ -4,18 +4,19 @@ namespace StellaOps.Scanner.Storage.Repositories;
public interface IReachabilityDriftResultRepository
{
Task StoreAsync(ReachabilityDriftResult result, CancellationToken ct = default);
Task StoreAsync(ReachabilityDriftResult result, CancellationToken ct = default, string? tenantId = null);
Task<ReachabilityDriftResult?> TryGetLatestForHeadAsync(string headScanId, string language, CancellationToken ct = default);
Task<ReachabilityDriftResult?> TryGetLatestForHeadAsync(string headScanId, string language, CancellationToken ct = default, string? tenantId = null);
Task<ReachabilityDriftResult?> TryGetByIdAsync(Guid driftId, CancellationToken ct = default);
Task<ReachabilityDriftResult?> TryGetByIdAsync(Guid driftId, CancellationToken ct = default, string? tenantId = null);
Task<bool> ExistsAsync(Guid driftId, CancellationToken ct = default);
Task<bool> ExistsAsync(Guid driftId, CancellationToken ct = default, string? tenantId = null);
Task<IReadOnlyList<DriftedSink>> ListSinksAsync(
Guid driftId,
DriftDirection direction,
int offset,
int limit,
CancellationToken ct = default);
CancellationToken ct = default,
string? tenantId = null);
}

5
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/IReachabilityResultRepository.cs
View File

@@ -4,8 +4,7 @@ namespace StellaOps.Scanner.Storage.Repositories;
public interface IReachabilityResultRepository
{
Task StoreAsync(ReachabilityAnalysisResult result, CancellationToken ct = default);
Task StoreAsync(ReachabilityAnalysisResult result, CancellationToken ct = default, string? tenantId = null);
Task<ReachabilityAnalysisResult?> TryGetLatestAsync(string scanId, string language, CancellationToken ct = default);
Task<ReachabilityAnalysisResult?> TryGetLatestAsync(string scanId, string language, CancellationToken ct = default, string? tenantId = null);
}

16
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/IScanMetricsRepository.cs
View File

@@ -32,12 +32,18 @@ public interface IScanMetricsRepository
/// <summary>
/// Get metrics by scan ID.
/// </summary>
Task<ScanMetrics?> GetByScanIdAsync(Guid scanId, CancellationToken cancellationToken = default);
Task<ScanMetrics?> GetByScanIdAsync(
Guid tenantId,
Guid scanId,
CancellationToken cancellationToken = default);
/// <summary>
/// Get metrics by metrics ID.
/// </summary>
Task<ScanMetrics?> GetByIdAsync(Guid metricsId, CancellationToken cancellationToken = default);
Task<ScanMetrics?> GetByIdAsync(
Guid tenantId,
Guid metricsId,
CancellationToken cancellationToken = default);
/// <summary>
/// Get execution phases for a scan.
@@ -75,11 +81,15 @@ public interface IScanMetricsRepository
/// Get scans by artifact digest.
/// </summary>
Task<IReadOnlyList<ScanMetrics>> GetByArtifactAsync(
Guid tenantId,
string artifactDigest,
CancellationToken cancellationToken = default);
/// <summary>
/// Delete old metrics (for retention).
/// </summary>
Task<int> DeleteOlderThanAsync(DateTimeOffset threshold, CancellationToken cancellationToken = default);
Task<int> DeleteOlderThanAsync(
Guid tenantId,
DateTimeOffset threshold,
CancellationToken cancellationToken = default);
}

3
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/ISecretDetectionSettingsRepository.cs
View File

@@ -60,6 +60,7 @@ public interface ISecretExceptionPatternRepository
/// Gets a specific exception pattern.
/// </summary>
Task<SecretExceptionPatternRow?> GetByIdAsync(
Guid tenantId,
Guid exceptionId,
CancellationToken cancellationToken = default);
@@ -74,6 +75,7 @@ public interface ISecretExceptionPatternRepository
/// Updates an exception pattern.
/// </summary>
Task<bool> UpdateAsync(
Guid tenantId,
SecretExceptionPatternRow pattern,
CancellationToken cancellationToken = default);
@@ -81,6 +83,7 @@ public interface ISecretExceptionPatternRepository
/// Deletes an exception pattern.
/// </summary>
Task<bool> DeleteAsync(
Guid tenantId,
Guid exceptionId,
CancellationToken cancellationToken = default);

31
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/PostgresScanMetricsRepository.cs
View File

@@ -166,13 +166,18 @@ public sealed class PostgresScanMetricsRepository : IScanMetricsRepository
}
/// <inheritdoc/>
public async Task<ScanMetrics?> GetByScanIdAsync(Guid scanId, CancellationToken cancellationToken = default)
public async Task<ScanMetrics?> GetByScanIdAsync(
Guid tenantId,
Guid scanId,
CancellationToken cancellationToken = default)
{
const string sql = """
SELECT * FROM scanner.scan_metrics WHERE scan_id = @scanId
SELECT * FROM scanner.scan_metrics
WHERE tenant_id = @tenantId AND scan_id = @scanId
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("tenantId", tenantId);
cmd.Parameters.AddWithValue("scanId", scanId);
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
@@ -185,13 +190,18 @@ public sealed class PostgresScanMetricsRepository : IScanMetricsRepository
}
/// <inheritdoc/>
public async Task<ScanMetrics?> GetByIdAsync(Guid metricsId, CancellationToken cancellationToken = default)
public async Task<ScanMetrics?> GetByIdAsync(
Guid tenantId,
Guid metricsId,
CancellationToken cancellationToken = default)
{
const string sql = """
SELECT * FROM scanner.scan_metrics WHERE metrics_id = @metricsId
SELECT * FROM scanner.scan_metrics
WHERE tenant_id = @tenantId AND metrics_id = @metricsId
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("tenantId", tenantId);
cmd.Parameters.AddWithValue("metricsId", metricsId);
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
@@ -309,16 +319,18 @@ public sealed class PostgresScanMetricsRepository : IScanMetricsRepository
/// <inheritdoc/>
public async Task<IReadOnlyList<ScanMetrics>> GetByArtifactAsync(
Guid tenantId,
string artifactDigest,
CancellationToken cancellationToken = default)
{
const string sql = """
SELECT * FROM scanner.scan_metrics
WHERE artifact_digest = @artifactDigest
WHERE tenant_id = @tenantId AND artifact_digest = @artifactDigest
ORDER BY started_at DESC
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("tenantId", tenantId);
cmd.Parameters.AddWithValue("artifactDigest", artifactDigest);
var metrics = new List<ScanMetrics>();
@@ -333,13 +345,18 @@ public sealed class PostgresScanMetricsRepository : IScanMetricsRepository
}
/// <inheritdoc/>
public async Task<int> DeleteOlderThanAsync(DateTimeOffset threshold, CancellationToken cancellationToken = default)
public async Task<int> DeleteOlderThanAsync(
Guid tenantId,
DateTimeOffset threshold,
CancellationToken cancellationToken = default)
{
const string sql = """
DELETE FROM scanner.scan_metrics WHERE started_at < @threshold
DELETE FROM scanner.scan_metrics
WHERE tenant_id = @tenantId AND started_at < @threshold
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("tenantId", tenantId);
cmd.Parameters.AddWithValue("threshold", threshold);
return await cmd.ExecuteNonQueryAsync(cancellationToken);

8
src/Scanner/__Libraries/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj
View File

@@ -8,13 +8,19 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" />
<PackageReference Include="Dapper" />
<PackageReference Include="Microsoft.EntityFrameworkCore" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" PrivateAssets="all" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Npgsql" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" />
</ItemGroup>
<ItemGroup>
<!-- Prevent automatic compiled-model binding so non-default schemas can build runtime models. -->
<Compile Remove="EfCore\CompiledModels\ScannerDbContextAssemblyAttributes.cs" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Postgres\Migrations\**\*.sql" LogicalName="%(RecursiveDir)%(Filename)%(Extension)" />

8
src/Scanner/__Libraries/StellaOps.Scanner.Storage/TASKS.md
View File

@@ -9,3 +9,11 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| HOT-002 | DONE | `SPRINT_20260210_001_DOCS_sbom_attestation_hot_lookup_contract.md`: added `scanner.artifact_boms` partitioned schema + indexes + helper functions. |
| HOT-003 | DONE | `SPRINT_20260210_001_DOCS_sbom_attestation_hot_lookup_contract.md`: implemented ingestion projection and idempotent upsert flow. |
| HOT-005 | DONE | `SPRINT_20260210_001_DOCS_sbom_attestation_hot_lookup_contract.md`: delivered partition pre-create and retention maintenance jobs/assets. |
| SPRINT-20260222-057-SCAN-TEN-11 | DONE | `SPRINT_20260222_057_Scanner_tenant_isolation_for_scans_triage_webhooks.md`: replaced fixed-tenant SmartDiff/Reachability repository scopes with resolved tenant context mapping for tenant-partitioned tables (2026-02-23). |
| SPRINT-20260222-057-SCAN-TEN-12 | DONE | `SPRINT_20260222_057_Scanner_tenant_isolation_for_scans_triage_webhooks.md`: removed remaining fixed-tenant adapters for `risk_state_snapshots` and `reachability_results` by tenant-parameterizing repository contracts and Postgres adapters (2026-02-23). |
| SPRINT-20260222-057-SCAN-TEN-13 | DONE | `SPRINT_20260222_057_Scanner_tenant_isolation_for_scans_triage_webhooks.md`: enforced tenant predicates for `secret_exception_pattern` get/update/delete repository paths and removed ID-only tenant-agnostic operations (2026-02-23). |
| SCAN-EF-01 | DONE | `SPRINT_20260222_095_Scanner_dal_to_efcore.md`: Verified AGENTS.md and migration registry wiring (2026-02-23). |
| SCAN-EF-02 | DONE | `SPRINT_20260222_095_Scanner_dal_to_efcore.md`: Scaffolded EF Core model baseline - ScannerDbContext, 13 entity models, design-time factory, compiled model stubs, ScannerDbContextFactory (2026-02-23). |
| SCAN-EF-03 | DONE | `SPRINT_20260222_095_Scanner_dal_to_efcore.md`: Converted all Dapper repositories to EF Core. Removed Dapper package. Build 0 errors 0 warnings (2026-02-23). |
| SCAN-EF-04 | DONE | `SPRINT_20260222_095_Scanner_dal_to_efcore.md`: Compiled model and runtime static model path verified (2026-02-23). |
| SCAN-EF-05 | DONE | `SPRINT_20260222_095_Scanner_dal_to_efcore.md`: Sequential build validation passed. Sprint docs updated (2026-02-23). |

6
src/Scanner/__Libraries/StellaOps.Scanner.Triage/Entities/TriageCaseCurrent.cs
View File

@@ -17,6 +17,12 @@ public sealed class TriageCaseCurrent
[Column("case_id")]
public Guid CaseId { get; init; }
/// <summary>
/// Tenant owning this case.
/// </summary>
[Column("tenant_id")]
public string TenantId { get; init; } = string.Empty;
/// <summary>
/// The asset ID.
/// </summary>

7
src/Scanner/__Libraries/StellaOps.Scanner.Triage/Entities/TriageFinding.cs
View File

@@ -18,6 +18,13 @@ public sealed class TriageFinding
[Column("id")]
public required Guid Id { get; init; }
/// <summary>
/// Tenant that owns this finding.
/// </summary>
[Required]
[Column("tenant_id")]
public required string TenantId { get; init; }
/// <summary>
/// The asset this finding applies to.
/// </summary>

7
src/Scanner/__Libraries/StellaOps.Scanner.Triage/Entities/TriageScan.cs
View File

@@ -16,6 +16,13 @@ public sealed class TriageScan
[Column("id")]
public required Guid Id { get; init; }
/// <summary>
/// Tenant that owns this scan.
/// </summary>
[Required]
[Column("tenant_id")]
public required string TenantId { get; init; }
/// <summary>
/// Image reference that was scanned.
/// </summary>

13
src/Scanner/__Libraries/StellaOps.Scanner.Triage/Migrations/V3700_001__triage_schema.sql
View File

@@ -65,6 +65,7 @@ END $$;
-- Scan metadata
CREATE TABLE IF NOT EXISTS triage_scan (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id text NOT NULL DEFAULT 'default',
image_reference text NOT NULL,
image_digest text NULL,
target_digest text NULL,
@@ -86,6 +87,7 @@ CREATE TABLE IF NOT EXISTS triage_scan (
-- Core: finding (caseId == findingId)
CREATE TABLE IF NOT EXISTS triage_finding (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id text NOT NULL DEFAULT 'default',
asset_id uuid NOT NULL,
environment_id uuid NULL,
asset_label text NOT NULL,
@@ -104,14 +106,18 @@ CREATE TABLE IF NOT EXISTS triage_finding (
superseded_by text NULL,
delta_comparison_id uuid NULL,
knowledge_snapshot_id text NULL,
UNIQUE (asset_id, environment_id, purl, cve_id, rule_id)
UNIQUE (tenant_id, asset_id, environment_id, purl, cve_id, rule_id)
);
CREATE INDEX IF NOT EXISTS ix_triage_scan_tenant_id ON triage_scan (tenant_id);
CREATE INDEX IF NOT EXISTS ix_triage_finding_tenant_id ON triage_finding (tenant_id);
CREATE INDEX IF NOT EXISTS ix_triage_finding_last_seen ON triage_finding (last_seen_at DESC);
CREATE INDEX IF NOT EXISTS ix_triage_finding_asset_label ON triage_finding (asset_label);
CREATE INDEX IF NOT EXISTS ix_triage_finding_purl ON triage_finding (purl);
CREATE INDEX IF NOT EXISTS ix_triage_finding_cve ON triage_finding (cve_id);
ALTER TABLE triage_scan ADD COLUMN IF NOT EXISTS tenant_id text NOT NULL DEFAULT 'default';
ALTER TABLE triage_finding ADD COLUMN IF NOT EXISTS tenant_id text NOT NULL DEFAULT 'default';
ALTER TABLE triage_finding ADD COLUMN IF NOT EXISTS artifact_digest text NULL;
ALTER TABLE triage_finding ADD COLUMN IF NOT EXISTS scan_id uuid NULL;
ALTER TABLE triage_finding ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT now();
@@ -122,6 +128,9 @@ ALTER TABLE triage_finding ADD COLUMN IF NOT EXISTS fixed_in_version text NULL;
ALTER TABLE triage_finding ADD COLUMN IF NOT EXISTS superseded_by text NULL;
ALTER TABLE triage_finding ADD COLUMN IF NOT EXISTS delta_comparison_id uuid NULL;
ALTER TABLE triage_finding ADD COLUMN IF NOT EXISTS knowledge_snapshot_id text NULL;
ALTER TABLE triage_finding DROP CONSTRAINT IF EXISTS triage_finding_asset_id_environment_id_purl_cve_id_rule_id_key;
CREATE UNIQUE INDEX IF NOT EXISTS ux_triage_finding_tenant_asset_env_purl_cve_rule
ON triage_finding (tenant_id, asset_id, environment_id, purl, cve_id, rule_id);
DO $$
BEGIN
@@ -296,6 +305,7 @@ latest_vex AS (
)
SELECT
f.id AS case_id,
f.tenant_id,
f.asset_id,
f.environment_id,
f.asset_label,
@@ -323,4 +333,3 @@ FROM triage_finding f
LEFT JOIN latest_risk r ON r.finding_id = f.id
LEFT JOIN latest_reach re ON re.finding_id = f.id
LEFT JOIN latest_vex v ON v.finding_id = f.id;

1
src/Scanner/__Libraries/StellaOps.Scanner.Triage/TASKS.md
View File

@@ -7,3 +7,4 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| REMED-05 | TODO | Remediation checklist: docs/implplan/audits/csproj-standards/remediation/checklists/src/Scanner/__Libraries/StellaOps.Scanner.Triage/StellaOps.Scanner.Triage.md. |
| REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |
| SPRINT-20260208-063-TRIAGE-001 | DONE | Implement deterministic exploit-path grouping algorithm and triage cluster model wiring for sprint 063 (2026-02-08). |
| SPRINT-20260222-057-SCAN-TEN | DONE | `SPRINT_20260222_057_Scanner_tenant_isolation_for_scans_triage_webhooks.md`: added `tenant_id` discriminator fields and tenant-scoped triage uniqueness/indexing for SCAN-TEN-04 (2026-02-22). |

5
src/Scanner/__Libraries/StellaOps.Scanner.Triage/TriageDbContext.cs
View File

@@ -104,7 +104,10 @@ public sealed class TriageDbContext : DbContext
entity.HasIndex(e => e.CveId)
.HasDatabaseName("ix_triage_finding_cve");
entity.HasIndex(e => new { e.AssetId, e.EnvironmentId, e.Purl, e.CveId, e.RuleId })
entity.HasIndex(e => e.TenantId)
.HasDatabaseName("ix_triage_finding_tenant_id");
entity.HasIndex(e => new { e.TenantId, e.AssetId, e.EnvironmentId, e.Purl, e.CveId, e.RuleId })
.IsUnique();
});

2
src/Scanner/__Libraries/StellaOps.Scanner.VulnSurfaces.Tests/VulnSurfaceServiceTests.cs
View File

@@ -123,7 +123,7 @@ public sealed class VulnSurfaceServiceTests
public Task<IReadOnlyList<VulnSurface>> GetSurfacesByCveAsync(Guid tenantId, string cveId, CancellationToken cancellationToken = default)
=> Task.FromResult<IReadOnlyList<VulnSurface>>(Array.Empty<VulnSurface>());
public Task<bool> DeleteSurfaceAsync(Guid surfaceId, CancellationToken cancellationToken = default)
public Task<bool> DeleteSurfaceAsync(Guid tenantId, Guid surfaceId, CancellationToken cancellationToken = default)
=> Task.FromResult(false);
}

2
src/Scanner/__Libraries/StellaOps.Scanner.VulnSurfaces/Storage/IVulnSurfaceRepository.cs
View File

@@ -93,7 +93,7 @@ public interface IVulnSurfaceRepository
/// Deletes a vulnerability surface and all related data.
/// </summary>
Task<bool> DeleteSurfaceAsync(
Guid tenantId,
Guid surfaceId,
CancellationToken cancellationToken = default);
}

6
src/Scanner/__Libraries/StellaOps.Scanner.VulnSurfaces/Storage/PostgresVulnSurfaceRepository.cs
View File

@@ -322,17 +322,21 @@ public sealed class PostgresVulnSurfaceRepository : IVulnSurfaceRepository
}
public async Task<bool> DeleteSurfaceAsync(
Guid tenantId,
Guid surfaceId,
CancellationToken cancellationToken = default)
{
const string sql = """
DELETE FROM scanner.vuln_surfaces WHERE id = @id
DELETE FROM scanner.vuln_surfaces
WHERE tenant_id = @tenant_id AND id = @id
""";
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
await SetTenantContextAsync(connection, tenantId, cancellationToken);
await using var command = new NpgsqlCommand(sql, connection);
command.CommandTimeout = _commandTimeoutSeconds;
command.Parameters.AddWithValue("tenant_id", tenantId);
command.Parameters.AddWithValue("id", surfaceId);
var rows = await command.ExecuteNonQueryAsync(cancellationToken);