consolidation of some of the modules, localization fixes, product advisories work, qa work
This commit is contained in:
@@ -8,10 +8,12 @@ using System.Text.Json;
|
||||
namespace StellaOps.Unknowns.Persistence.EfCore.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// EF Core implementation of <see cref="IUnknownRepository"/>.
|
||||
/// Deprecated scaffold-only implementation of <see cref="IUnknownRepository"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is a placeholder implementation. After scaffolding, update to use the generated entities.
|
||||
/// This project path is intentionally non-active for runtime registrations.
|
||||
/// Active implementation: src/Unknowns/__Libraries/StellaOps.Unknowns.Persistence/EfCore/Repositories/UnknownEfRepository.cs
|
||||
/// This file remains scaffold-only to avoid drift while preserving historical scaffolding output.
|
||||
/// For complex queries (CTEs, window functions), use raw SQL via <see cref="UnknownsDbContext.RawSqlQueryAsync{T}"/>.
|
||||
/// </remarks>
|
||||
public sealed class UnknownEfRepository : IUnknownRepository
|
||||
@@ -288,7 +290,7 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
string? primarySuggestedAction,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
throw new NotImplementedException("Scaffold entities first");
|
||||
throw new NotSupportedException("Deprecated scaffold-only repository path. Use StellaOps.Unknowns.Persistence/EfCore/Repositories/UnknownEfRepository.cs.");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -298,7 +300,7 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
int? limit = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
throw new NotImplementedException("Scaffold entities first");
|
||||
throw new NotSupportedException("Deprecated scaffold-only repository path. Use StellaOps.Unknowns.Persistence/EfCore/Repositories/UnknownEfRepository.cs.");
|
||||
}
|
||||
|
||||
// Helper DTOs for raw SQL queries
|
||||
|
||||
@@ -9,6 +9,7 @@ using StellaOps.Unknowns.Persistence.Postgres;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Unknowns.Persistence.EfCore.Repositories;
|
||||
|
||||
@@ -19,6 +20,12 @@ namespace StellaOps.Unknowns.Persistence.EfCore.Repositories;
|
||||
/// </summary>
|
||||
public sealed class UnknownEfRepository : IUnknownRepository
|
||||
{
|
||||
private static readonly JsonSerializerOptions HintJsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
Converters = { new JsonStringEnumConverter() }
|
||||
};
|
||||
|
||||
private readonly UnknownsDataSource _dataSource;
|
||||
private readonly ILogger<UnknownEfRepository> _logger;
|
||||
private readonly int _commandTimeoutSeconds;
|
||||
@@ -74,7 +81,8 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
};
|
||||
|
||||
// Use raw SQL for INSERT to handle PostgreSQL enum casting
|
||||
await dbContext.Database.ExecuteSqlRawAsync(
|
||||
await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
INSERT INTO unknowns.unknown (
|
||||
id, tenant_id, subject_hash, subject_type, subject_ref,
|
||||
@@ -87,15 +95,15 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
{11}, {12}, {13}, {14}, {15}
|
||||
)
|
||||
""",
|
||||
cancellationToken,
|
||||
id, tenantId, subjectHash, MapSubjectType(subjectType), subjectRef,
|
||||
MapUnknownKind(kind),
|
||||
severity.HasValue ? MapSeverity(severity.Value) : (object)DBNull.Value,
|
||||
NullableParameter("severity", severity.HasValue ? MapSeverity(severity.Value) : null),
|
||||
context ?? "{}",
|
||||
sourceScanId.HasValue ? sourceScanId.Value : (object)DBNull.Value,
|
||||
sourceGraphId.HasValue ? sourceGraphId.Value : (object)DBNull.Value,
|
||||
(object?)sourceSbomDigest ?? DBNull.Value,
|
||||
now, now, now, createdBy, now,
|
||||
cancellationToken);
|
||||
NullableParameter("source_scan_id", sourceScanId),
|
||||
NullableParameter("source_graph_id", sourceGraphId),
|
||||
NullableParameter("source_sbom_digest", sourceSbomDigest),
|
||||
now, now, now, createdBy, now);
|
||||
|
||||
_logger.LogDebug("Created unknown {Id} for tenant {TenantId}, kind={Kind}", id, tenantId, kind);
|
||||
|
||||
@@ -283,7 +291,8 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await dbContext.Database.ExecuteSqlRawAsync(
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET resolved_at = {0},
|
||||
@@ -296,15 +305,15 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
AND id = {7}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
cancellationToken,
|
||||
now,
|
||||
MapResolutionType(resolutionType),
|
||||
(object?)resolutionRef ?? DBNull.Value,
|
||||
(object?)resolutionNotes ?? DBNull.Value,
|
||||
NullableParameter("resolution_ref", resolutionRef),
|
||||
NullableParameter("resolution_notes", resolutionNotes),
|
||||
now,
|
||||
now,
|
||||
tenantId,
|
||||
id,
|
||||
cancellationToken);
|
||||
id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
@@ -328,7 +337,8 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await dbContext.Database.ExecuteSqlRawAsync(
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET sys_to = {0},
|
||||
@@ -337,8 +347,8 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
AND id = {3}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
now, now, tenantId, id,
|
||||
cancellationToken);
|
||||
cancellationToken,
|
||||
now, now, tenantId, id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
@@ -478,7 +488,8 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await dbContext.Database.ExecuteSqlRawAsync(
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET popularity_score = {0},
|
||||
@@ -501,15 +512,15 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
AND id = {17}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
cancellationToken,
|
||||
popularityScore, deploymentCount, exploitPotentialScore, uncertaintyScore,
|
||||
uncertaintyFlags ?? "{}",
|
||||
centralityScore, degreeCentrality, betweennessCentrality,
|
||||
stalenessScore, daysSinceAnalysis, compositeScore,
|
||||
MapTriageBand(triageBand),
|
||||
scoringTrace ?? "{}",
|
||||
nextScheduledRescan.HasValue ? nextScheduledRescan.Value : (object)DBNull.Value,
|
||||
now, now, tenantId, id,
|
||||
cancellationToken);
|
||||
NullableParameter("next_scheduled_rescan", nextScheduledRescan),
|
||||
now, now, tenantId, id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
@@ -534,7 +545,8 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await dbContext.Database.ExecuteSqlRawAsync(
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET rescan_attempts = rescan_attempts + 1,
|
||||
@@ -545,10 +557,10 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
AND id = {4}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
cancellationToken,
|
||||
result,
|
||||
nextRescan.HasValue ? nextRescan.Value : (object)DBNull.Value,
|
||||
now, tenantId, id,
|
||||
cancellationToken);
|
||||
NullableParameter("next_rescan", nextRescan),
|
||||
now, tenantId, id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
@@ -615,7 +627,7 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public Task<Unknown> AttachProvenanceHintsAsync(
|
||||
public async Task<Unknown> AttachProvenanceHintsAsync(
|
||||
string tenantId,
|
||||
Guid id,
|
||||
IReadOnlyList<ProvenanceHint> hints,
|
||||
@@ -624,18 +636,98 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
string? primarySuggestedAction,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// TODO: Implement provenance hints storage when migration 002 table name discrepancy is resolved
|
||||
throw new NotImplementedException("Provenance hints storage not yet implemented");
|
||||
if (hints is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(hints));
|
||||
}
|
||||
|
||||
if (combinedConfidence is < 0 or > 1)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(combinedConfidence), "Combined confidence must be between 0 and 1.");
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var sortedHints = hints
|
||||
.OrderByDescending(h => h.Confidence)
|
||||
.ThenBy(h => h.HintId, StringComparer.Ordinal)
|
||||
.ThenBy(h => h.GeneratedAt)
|
||||
.ToArray();
|
||||
var hintsJson = JsonSerializer.Serialize(sortedHints, HintJsonOptions);
|
||||
var normalizedConfidence = combinedConfidence.HasValue
|
||||
? decimal.Round((decimal)combinedConfidence.Value, 4, MidpointRounding.AwayFromZero)
|
||||
: (decimal?)null;
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET provenance_hints = {0}::jsonb,
|
||||
best_hypothesis = {1},
|
||||
combined_confidence = {2},
|
||||
primary_suggested_action = {3},
|
||||
updated_at = {4}
|
||||
WHERE tenant_id = {5}
|
||||
AND id = {6}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
cancellationToken,
|
||||
hintsJson,
|
||||
NullableParameter("best_hypothesis", bestHypothesis),
|
||||
NullableParameter("combined_confidence", normalizedConfidence),
|
||||
NullableParameter("primary_suggested_action", primarySuggestedAction),
|
||||
now,
|
||||
tenantId,
|
||||
id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Unknown {id} not found or already superseded.");
|
||||
}
|
||||
|
||||
var updated = await GetByIdAsync(tenantId, id, cancellationToken);
|
||||
return updated ?? throw new InvalidOperationException($"Failed to retrieve unknown {id} after provenance hint update.");
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<Unknown>> GetWithHighConfidenceHintsAsync(
|
||||
public async Task<IReadOnlyList<Unknown>> GetWithHighConfidenceHintsAsync(
|
||||
string tenantId,
|
||||
double minConfidence = 0.7,
|
||||
int? limit = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// TODO: Implement provenance hints query when migration 002 table name discrepancy is resolved
|
||||
throw new NotImplementedException("Provenance hints query not yet implemented");
|
||||
if (minConfidence is < 0 or > 1)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(minConfidence), "Minimum confidence must be between 0 and 1.");
|
||||
}
|
||||
|
||||
if (limit.HasValue && limit.Value <= 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "reader", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
IQueryable<UnknownEntity> query = dbContext.Unknowns
|
||||
.AsNoTracking()
|
||||
.Where(e =>
|
||||
e.TenantId == tenantId
|
||||
&& e.ValidTo == null
|
||||
&& e.SysTo == null
|
||||
&& e.CombinedConfidence.HasValue
|
||||
&& e.CombinedConfidence.Value >= (decimal)minConfidence)
|
||||
.OrderByDescending(e => e.CombinedConfidence)
|
||||
.ThenBy(e => e.Id);
|
||||
|
||||
if (limit.HasValue)
|
||||
{
|
||||
query = query.Take(limit.Value);
|
||||
}
|
||||
|
||||
var entities = await query.ToListAsync(cancellationToken);
|
||||
return entities.Select(MapToDomain).ToList();
|
||||
}
|
||||
|
||||
private string GetSchemaName() => UnknownsDataSource.DefaultSchemaName;
|
||||
@@ -687,10 +779,41 @@ public sealed class UnknownEfRepository : IUnknownRepository
|
||||
NextScheduledRescan = entity.NextScheduledRescan,
|
||||
LastAnalyzedAt = entity.LastAnalyzedAt,
|
||||
EvidenceSetHash = entity.EvidenceSetHash,
|
||||
GraphSliceHash = entity.GraphSliceHash
|
||||
GraphSliceHash = entity.GraphSliceHash,
|
||||
ProvenanceHints = ParseHints(entity.ProvenanceHints),
|
||||
BestHypothesis = entity.BestHypothesis,
|
||||
CombinedConfidence = entity.CombinedConfidence.HasValue ? (double)entity.CombinedConfidence.Value : null,
|
||||
PrimarySuggestedAction = entity.PrimarySuggestedAction
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ProvenanceHint> ParseHints(string? hintsJson)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(hintsJson) || hintsJson == "[]")
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize<IReadOnlyList<ProvenanceHint>>(hintsJson, HintJsonOptions) ?? [];
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private static NpgsqlParameter NullableParameter(string name, object? value)
|
||||
=> new(name, value ?? DBNull.Value);
|
||||
|
||||
private static Task<int> ExecuteSqlAsync(
|
||||
UnknownsDbContext dbContext,
|
||||
string sql,
|
||||
CancellationToken cancellationToken,
|
||||
params object[] parameters)
|
||||
=> dbContext.Database.ExecuteSqlRawAsync(sql, parameters, cancellationToken);
|
||||
|
||||
private static string ComputeSubjectHash(string subjectRef)
|
||||
{
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(subjectRef));
|
||||
|
||||
@@ -15,22 +15,22 @@ BEGIN;
|
||||
-- Step 1: Add provenance hint columns to unknowns table
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE IF EXISTS unknowns.unknowns
|
||||
ALTER TABLE IF EXISTS unknowns.unknown
|
||||
ADD COLUMN IF NOT EXISTS provenance_hints JSONB DEFAULT '[]'::jsonb NOT NULL,
|
||||
ADD COLUMN IF NOT EXISTS best_hypothesis TEXT,
|
||||
ADD COLUMN IF NOT EXISTS combined_confidence NUMERIC(4,4) CHECK (combined_confidence IS NULL OR (combined_confidence >= 0 AND combined_confidence <= 1)),
|
||||
ADD COLUMN IF NOT EXISTS primary_suggested_action TEXT;
|
||||
|
||||
COMMENT ON COLUMN unknowns.unknowns.provenance_hints IS
|
||||
COMMENT ON COLUMN unknowns.unknown.provenance_hints IS
|
||||
'Array of structured provenance hints (ProvenanceHint records)';
|
||||
|
||||
COMMENT ON COLUMN unknowns.unknowns.best_hypothesis IS
|
||||
COMMENT ON COLUMN unknowns.unknown.best_hypothesis IS
|
||||
'Best hypothesis from all hints (highest confidence)';
|
||||
|
||||
COMMENT ON COLUMN unknowns.unknowns.combined_confidence IS
|
||||
COMMENT ON COLUMN unknowns.unknown.combined_confidence IS
|
||||
'Combined confidence score from all hints (0.0 - 1.0)';
|
||||
|
||||
COMMENT ON COLUMN unknowns.unknowns.primary_suggested_action IS
|
||||
COMMENT ON COLUMN unknowns.unknown.primary_suggested_action IS
|
||||
'Primary suggested action (highest priority)';
|
||||
|
||||
-- ============================================================================
|
||||
@@ -38,7 +38,7 @@ COMMENT ON COLUMN unknowns.unknowns.primary_suggested_action IS
|
||||
-- ============================================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_unknowns_provenance_hints_gin
|
||||
ON unknowns.unknowns USING GIN (provenance_hints);
|
||||
ON unknowns.unknown USING GIN (provenance_hints);
|
||||
|
||||
COMMENT ON INDEX unknowns.idx_unknowns_provenance_hints_gin IS
|
||||
'GIN index for efficient JSONB queries on provenance hints';
|
||||
@@ -48,7 +48,7 @@ COMMENT ON INDEX unknowns.idx_unknowns_provenance_hints_gin IS
|
||||
-- ============================================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_unknowns_combined_confidence
|
||||
ON unknowns.unknowns (tenant_id, combined_confidence DESC)
|
||||
ON unknowns.unknown (tenant_id, combined_confidence DESC)
|
||||
WHERE combined_confidence IS NOT NULL AND combined_confidence >= 0.7;
|
||||
|
||||
COMMENT ON INDEX unknowns.idx_unknowns_combined_confidence IS
|
||||
@@ -94,8 +94,18 @@ COMMENT ON FUNCTION unknowns.validate_provenance_hints IS
|
||||
-- Step 5: Add validation constraint
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE IF EXISTS unknowns.unknowns
|
||||
ADD CONSTRAINT chk_provenance_hints_valid
|
||||
CHECK (unknowns.validate_provenance_hints(provenance_hints));
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'chk_provenance_hints_valid'
|
||||
) THEN
|
||||
ALTER TABLE unknowns.unknown
|
||||
ADD CONSTRAINT chk_provenance_hints_valid
|
||||
CHECK (unknowns.validate_provenance_hints(provenance_hints));
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
COMMIT;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
using StellaOps.Unknowns.Core.Models;
|
||||
using StellaOps.Unknowns.Core.Repositories;
|
||||
using StellaOps.Unknowns.Persistence.EfCore.Context;
|
||||
@@ -8,6 +9,7 @@ using StellaOps.Unknowns.Persistence.EfCore.Models;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Unknowns.Persistence.Postgres.Repositories;
|
||||
|
||||
@@ -18,6 +20,12 @@ namespace StellaOps.Unknowns.Persistence.Postgres.Repositories;
|
||||
/// </summary>
|
||||
public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
{
|
||||
private static readonly JsonSerializerOptions HintJsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
Converters = { new JsonStringEnumConverter() }
|
||||
};
|
||||
|
||||
private readonly UnknownsDataSource _dataSource;
|
||||
private readonly ILogger<PostgresUnknownRepository> _logger;
|
||||
private readonly int _commandTimeoutSeconds;
|
||||
@@ -53,7 +61,8 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
// Use raw SQL for INSERT to handle PostgreSQL enum casting
|
||||
await dbContext.Database.ExecuteSqlRawAsync(
|
||||
await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
INSERT INTO unknowns.unknown (
|
||||
id, tenant_id, subject_hash, subject_type, subject_ref,
|
||||
@@ -66,15 +75,15 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
{11}, {12}, {13}, {14}, {15}
|
||||
)
|
||||
""",
|
||||
cancellationToken,
|
||||
id, tenantId, subjectHash, MapSubjectType(subjectType), subjectRef,
|
||||
MapUnknownKind(kind),
|
||||
severity.HasValue ? MapSeverity(severity.Value) : (object)DBNull.Value,
|
||||
NullableParameter("severity", severity.HasValue ? MapSeverity(severity.Value) : null),
|
||||
context ?? "{}",
|
||||
sourceScanId.HasValue ? sourceScanId.Value : (object)DBNull.Value,
|
||||
sourceGraphId.HasValue ? sourceGraphId.Value : (object)DBNull.Value,
|
||||
(object?)sourceSbomDigest ?? DBNull.Value,
|
||||
now, now, now, createdBy, now,
|
||||
cancellationToken);
|
||||
NullableParameter("source_scan_id", sourceScanId),
|
||||
NullableParameter("source_graph_id", sourceGraphId),
|
||||
NullableParameter("source_sbom_digest", sourceSbomDigest),
|
||||
now, now, now, createdBy, now);
|
||||
|
||||
_logger.LogDebug("Created unknown {Id} for tenant {TenantId}, kind={Kind}", id, tenantId, kind);
|
||||
|
||||
@@ -268,7 +277,8 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await dbContext.Database.ExecuteSqlRawAsync(
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET resolved_at = {0},
|
||||
@@ -281,15 +291,15 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
AND id = {7}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
cancellationToken,
|
||||
now,
|
||||
MapResolutionType(resolutionType),
|
||||
(object?)resolutionRef ?? DBNull.Value,
|
||||
(object?)resolutionNotes ?? DBNull.Value,
|
||||
NullableParameter("resolution_ref", resolutionRef),
|
||||
NullableParameter("resolution_notes", resolutionNotes),
|
||||
now,
|
||||
now,
|
||||
tenantId,
|
||||
id,
|
||||
cancellationToken);
|
||||
id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
@@ -313,7 +323,8 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await dbContext.Database.ExecuteSqlRawAsync(
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET sys_to = {0},
|
||||
@@ -322,8 +333,8 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
AND id = {3}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
now, now, tenantId, id,
|
||||
cancellationToken);
|
||||
cancellationToken,
|
||||
now, now, tenantId, id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
@@ -463,7 +474,8 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await dbContext.Database.ExecuteSqlRawAsync(
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET popularity_score = {0},
|
||||
@@ -486,15 +498,15 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
AND id = {17}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
cancellationToken,
|
||||
popularityScore, deploymentCount, exploitPotentialScore, uncertaintyScore,
|
||||
uncertaintyFlags ?? "{}",
|
||||
centralityScore, degreeCentrality, betweennessCentrality,
|
||||
stalenessScore, daysSinceAnalysis, compositeScore,
|
||||
MapTriageBand(triageBand),
|
||||
scoringTrace ?? "{}",
|
||||
nextScheduledRescan.HasValue ? nextScheduledRescan.Value : (object)DBNull.Value,
|
||||
now, now, tenantId, id,
|
||||
cancellationToken);
|
||||
NullableParameter("next_scheduled_rescan", nextScheduledRescan),
|
||||
now, now, tenantId, id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
@@ -519,7 +531,8 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await dbContext.Database.ExecuteSqlRawAsync(
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET rescan_attempts = rescan_attempts + 1,
|
||||
@@ -530,10 +543,10 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
AND id = {4}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
cancellationToken,
|
||||
result,
|
||||
nextRescan.HasValue ? nextRescan.Value : (object)DBNull.Value,
|
||||
now, tenantId, id,
|
||||
cancellationToken);
|
||||
NullableParameter("next_rescan", nextRescan),
|
||||
now, tenantId, id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
@@ -600,7 +613,7 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
public Task<Unknown> AttachProvenanceHintsAsync(
|
||||
public async Task<Unknown> AttachProvenanceHintsAsync(
|
||||
string tenantId,
|
||||
Guid id,
|
||||
IReadOnlyList<ProvenanceHint> hints,
|
||||
@@ -609,18 +622,98 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
string? primarySuggestedAction,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// TODO: Implement provenance hints storage
|
||||
throw new NotImplementedException("Provenance hints storage not yet implemented");
|
||||
if (hints is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(hints));
|
||||
}
|
||||
|
||||
if (combinedConfidence is < 0 or > 1)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(combinedConfidence), "Combined confidence must be between 0 and 1.");
|
||||
}
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var sortedHints = hints
|
||||
.OrderByDescending(h => h.Confidence)
|
||||
.ThenBy(h => h.HintId, StringComparer.Ordinal)
|
||||
.ThenBy(h => h.GeneratedAt)
|
||||
.ToArray();
|
||||
var hintsJson = JsonSerializer.Serialize(sortedHints, HintJsonOptions);
|
||||
var normalizedConfidence = combinedConfidence.HasValue
|
||||
? decimal.Round((decimal)combinedConfidence.Value, 4, MidpointRounding.AwayFromZero)
|
||||
: (decimal?)null;
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "writer", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var affected = await ExecuteSqlAsync(
|
||||
dbContext,
|
||||
"""
|
||||
UPDATE unknowns.unknown
|
||||
SET provenance_hints = {0}::jsonb,
|
||||
best_hypothesis = {1},
|
||||
combined_confidence = {2},
|
||||
primary_suggested_action = {3},
|
||||
updated_at = {4}
|
||||
WHERE tenant_id = {5}
|
||||
AND id = {6}
|
||||
AND sys_to IS NULL
|
||||
""",
|
||||
cancellationToken,
|
||||
hintsJson,
|
||||
NullableParameter("best_hypothesis", bestHypothesis),
|
||||
NullableParameter("combined_confidence", normalizedConfidence),
|
||||
NullableParameter("primary_suggested_action", primarySuggestedAction),
|
||||
now,
|
||||
tenantId,
|
||||
id);
|
||||
|
||||
if (affected == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Unknown {id} not found or already superseded.");
|
||||
}
|
||||
|
||||
var updated = await GetByIdAsync(tenantId, id, cancellationToken);
|
||||
return updated ?? throw new InvalidOperationException($"Failed to retrieve unknown {id} after provenance hint update.");
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<Unknown>> GetWithHighConfidenceHintsAsync(
|
||||
public async Task<IReadOnlyList<Unknown>> GetWithHighConfidenceHintsAsync(
|
||||
string tenantId,
|
||||
double minConfidence = 0.7,
|
||||
int? limit = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// TODO: Implement provenance hints query
|
||||
throw new NotImplementedException("Provenance hints query not yet implemented");
|
||||
if (minConfidence is < 0 or > 1)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(minConfidence), "Minimum confidence must be between 0 and 1.");
|
||||
}
|
||||
|
||||
if (limit.HasValue && limit.Value <= 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
await using var connection = await _dataSource.OpenConnectionAsync(tenantId, "reader", cancellationToken);
|
||||
await using var dbContext = UnknownsDbContextFactory.Create(connection, _commandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
IQueryable<UnknownEntity> query = dbContext.Unknowns
|
||||
.AsNoTracking()
|
||||
.Where(e =>
|
||||
e.TenantId == tenantId
|
||||
&& e.ValidTo == null
|
||||
&& e.SysTo == null
|
||||
&& e.CombinedConfidence.HasValue
|
||||
&& e.CombinedConfidence.Value >= (decimal)minConfidence)
|
||||
.OrderByDescending(e => e.CombinedConfidence)
|
||||
.ThenBy(e => e.Id);
|
||||
|
||||
if (limit.HasValue)
|
||||
{
|
||||
query = query.Take(limit.Value);
|
||||
}
|
||||
|
||||
var entities = await query.ToListAsync(cancellationToken);
|
||||
return entities.Select(MapToDomain).ToList();
|
||||
}
|
||||
|
||||
private static string GetSchemaName() => UnknownsDataSource.DefaultSchemaName;
|
||||
@@ -672,10 +765,41 @@ public sealed class PostgresUnknownRepository : IUnknownRepository
|
||||
NextScheduledRescan = entity.NextScheduledRescan,
|
||||
LastAnalyzedAt = entity.LastAnalyzedAt,
|
||||
EvidenceSetHash = entity.EvidenceSetHash,
|
||||
GraphSliceHash = entity.GraphSliceHash
|
||||
GraphSliceHash = entity.GraphSliceHash,
|
||||
ProvenanceHints = ParseHints(entity.ProvenanceHints),
|
||||
BestHypothesis = entity.BestHypothesis,
|
||||
CombinedConfidence = entity.CombinedConfidence.HasValue ? (double)entity.CombinedConfidence.Value : null,
|
||||
PrimarySuggestedAction = entity.PrimarySuggestedAction
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ProvenanceHint> ParseHints(string? hintsJson)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(hintsJson) || hintsJson == "[]")
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize<IReadOnlyList<ProvenanceHint>>(hintsJson, HintJsonOptions) ?? [];
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private static NpgsqlParameter NullableParameter(string name, object? value)
|
||||
=> new(name, value ?? DBNull.Value);
|
||||
|
||||
private static Task<int> ExecuteSqlAsync(
|
||||
UnknownsDbContext dbContext,
|
||||
string sql,
|
||||
CancellationToken cancellationToken,
|
||||
params object[] parameters)
|
||||
=> dbContext.Database.ExecuteSqlRawAsync(sql, parameters, cancellationToken);
|
||||
|
||||
private static string ComputeSubjectHash(string subjectRef)
|
||||
{
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(subjectRef));
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Npgsql;
|
||||
using StellaOps.Unknowns.Persistence.EfCore.CompiledModels;
|
||||
using StellaOps.Unknowns.Persistence.EfCore.Context;
|
||||
|
||||
namespace StellaOps.Unknowns.Persistence.Postgres;
|
||||
@@ -20,11 +19,8 @@ internal static class UnknownsDbContextFactory
|
||||
var optionsBuilder = new DbContextOptionsBuilder<UnknownsDbContext>()
|
||||
.UseNpgsql(connection, npgsql => npgsql.CommandTimeout(commandTimeoutSeconds));
|
||||
|
||||
if (string.Equals(normalizedSchema, UnknownsDataSource.DefaultSchemaName, StringComparison.Ordinal))
|
||||
{
|
||||
// Use the static compiled model when schema matches the default.
|
||||
optionsBuilder.UseModel(UnknownsDbContextModel.Instance);
|
||||
}
|
||||
// Disabled compiled-model binding until compiled model is regenerated in lockstep
|
||||
// with UnknownEntity mappings. Runtime model ensures schema/property parity.
|
||||
|
||||
return new UnknownsDbContext(optionsBuilder.Options, normalizedSchema);
|
||||
}
|
||||
|
||||
@@ -4,10 +4,13 @@ using Microsoft.Extensions.Options;
|
||||
using Npgsql;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
using StellaOps.Unknowns.Core.Models;
|
||||
using StellaOps.Unknowns.Core.Repositories;
|
||||
using StellaOps.Unknowns.Persistence.EfCore.Repositories;
|
||||
using StellaOps.Unknowns.Persistence.Postgres;
|
||||
using StellaOps.Unknowns.Persistence.Postgres.Repositories;
|
||||
using Testcontainers.PostgreSql;
|
||||
using Xunit;
|
||||
using System.Text.Json;
|
||||
|
||||
|
||||
using StellaOps.TestKit;
|
||||
@@ -21,6 +24,7 @@ public sealed class PostgresUnknownRepositoryTests : IAsyncLifetime
|
||||
|
||||
private UnknownsDataSource _dataSource = null!;
|
||||
private PostgresUnknownRepository _repository = null!;
|
||||
private UnknownEfRepository _efRepository = null!;
|
||||
private const string TestTenantId = "test-tenant";
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
@@ -43,6 +47,10 @@ public sealed class PostgresUnknownRepositoryTests : IAsyncLifetime
|
||||
_repository = new PostgresUnknownRepository(
|
||||
_dataSource,
|
||||
NullLogger<PostgresUnknownRepository>.Instance);
|
||||
|
||||
_efRepository = new UnknownEfRepository(
|
||||
_dataSource,
|
||||
NullLogger<UnknownEfRepository>.Instance);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
@@ -154,7 +162,11 @@ public sealed class PostgresUnknownRepositoryTests : IAsyncLifetime
|
||||
next_scheduled_rescan TIMESTAMPTZ,
|
||||
last_analyzed_at TIMESTAMPTZ,
|
||||
evidence_set_hash BYTEA,
|
||||
graph_slice_hash BYTEA
|
||||
graph_slice_hash BYTEA,
|
||||
provenance_hints JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
best_hypothesis TEXT,
|
||||
combined_confidence NUMERIC(4,4) CHECK (combined_confidence IS NULL OR (combined_confidence >= 0 AND combined_confidence <= 1)),
|
||||
primary_suggested_action TEXT
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_unknown_one_open_per_subject
|
||||
@@ -372,7 +384,185 @@ public sealed class PostgresUnknownRepositoryTests : IAsyncLifetime
|
||||
// After supersede, sys_to is set, so GetById (which filters sys_to IS NULL) returns null
|
||||
result.Should().BeNull();
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("postgres")]
|
||||
[InlineData("efcore")]
|
||||
public async Task AttachProvenanceHintsAsync_ShouldPersistHints_ForBothImplementations(string implementation)
|
||||
{
|
||||
var repository = GetRepository(implementation);
|
||||
var tenantId = $"tenant-hints-{implementation}";
|
||||
|
||||
var created = await repository.CreateAsync(
|
||||
tenantId,
|
||||
UnknownSubjectType.File,
|
||||
$"file:/usr/lib/{implementation}/libcrypto.so.3",
|
||||
UnknownKind.UnknownEcosystem,
|
||||
UnknownSeverity.High,
|
||||
"""{"path":"test"}""",
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
"test-user",
|
||||
CancellationToken.None);
|
||||
|
||||
var generatedAt = new DateTimeOffset(2026, 03, 04, 12, 00, 00, TimeSpan.Zero);
|
||||
var hints = CreateHints(generatedAt);
|
||||
|
||||
var updated = await repository.AttachProvenanceHintsAsync(
|
||||
tenantId,
|
||||
created.Id,
|
||||
hints,
|
||||
"Debian package candidate",
|
||||
0.93,
|
||||
"verify_build_id",
|
||||
CancellationToken.None);
|
||||
|
||||
updated.BestHypothesis.Should().Be("Debian package candidate");
|
||||
updated.CombinedConfidence.Should().BeApproximately(0.93, 0.0001);
|
||||
updated.PrimarySuggestedAction.Should().Be("verify_build_id");
|
||||
updated.ProvenanceHints.Should().HaveCount(2);
|
||||
updated.ProvenanceHints.Select(h => h.HintId)
|
||||
.Should().ContainInOrder("hint:sha256:aaa111", "hint:sha256:bbb222");
|
||||
|
||||
var reloaded = await repository.GetByIdAsync(tenantId, created.Id, CancellationToken.None);
|
||||
reloaded.Should().NotBeNull();
|
||||
reloaded!.ProvenanceHints.Should().HaveCount(2);
|
||||
reloaded.BestHypothesis.Should().Be("Debian package candidate");
|
||||
reloaded.CombinedConfidence.Should().BeApproximately(0.93, 0.0001);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("postgres")]
|
||||
[InlineData("efcore")]
|
||||
public async Task GetWithHighConfidenceHintsAsync_ShouldFilterTenantAndSortDeterministically(string implementation)
|
||||
{
|
||||
var repository = GetRepository(implementation);
|
||||
var tenantId = $"tenant-filter-{implementation}";
|
||||
var otherTenantId = $"{tenantId}-other";
|
||||
var now = new DateTimeOffset(2026, 03, 04, 12, 00, 00, TimeSpan.Zero);
|
||||
|
||||
var highA = await repository.CreateAsync(
|
||||
tenantId,
|
||||
UnknownSubjectType.Package,
|
||||
$"pkg:npm/high-a-{implementation}@1.0.0",
|
||||
UnknownKind.MissingFeed,
|
||||
UnknownSeverity.High,
|
||||
null, null, null, null, "test-user", CancellationToken.None);
|
||||
var highB = await repository.CreateAsync(
|
||||
tenantId,
|
||||
UnknownSubjectType.Package,
|
||||
$"pkg:npm/high-b-{implementation}@1.0.0",
|
||||
UnknownKind.MissingFeed,
|
||||
UnknownSeverity.High,
|
||||
null, null, null, null, "test-user", CancellationToken.None);
|
||||
var low = await repository.CreateAsync(
|
||||
tenantId,
|
||||
UnknownSubjectType.Package,
|
||||
$"pkg:npm/low-{implementation}@1.0.0",
|
||||
UnknownKind.MissingFeed,
|
||||
UnknownSeverity.Medium,
|
||||
null, null, null, null, "test-user", CancellationToken.None);
|
||||
var otherTenant = await repository.CreateAsync(
|
||||
otherTenantId,
|
||||
UnknownSubjectType.Package,
|
||||
$"pkg:npm/other-{implementation}@1.0.0",
|
||||
UnknownKind.MissingFeed,
|
||||
UnknownSeverity.High,
|
||||
null, null, null, null, "test-user", CancellationToken.None);
|
||||
|
||||
var hints = CreateHints(now);
|
||||
await repository.AttachProvenanceHintsAsync(tenantId, highA.Id, hints, "High A", 0.91, "verify_build_id", CancellationToken.None);
|
||||
await repository.AttachProvenanceHintsAsync(tenantId, highB.Id, hints, "High B", 0.91, "verify_build_id", CancellationToken.None);
|
||||
await repository.AttachProvenanceHintsAsync(tenantId, low.Id, hints, "Low", 0.72, "manual_triage", CancellationToken.None);
|
||||
await repository.AttachProvenanceHintsAsync(otherTenantId, otherTenant.Id, hints, "Other", 0.99, "manual_triage", CancellationToken.None);
|
||||
|
||||
var filtered = await repository.GetWithHighConfidenceHintsAsync(tenantId, minConfidence: 0.8, limit: null, CancellationToken.None);
|
||||
|
||||
filtered.Should().HaveCount(2);
|
||||
filtered.All(item => item.TenantId == tenantId).Should().BeTrue();
|
||||
filtered.All(item => item.CombinedConfidence >= 0.8).Should().BeTrue();
|
||||
|
||||
var expectedOrder = new[] { highA.Id, highB.Id }.OrderBy(id => id).ToArray();
|
||||
filtered.Select(item => item.Id).Should().ContainInOrder(expectedOrder);
|
||||
|
||||
var limited = await repository.GetWithHighConfidenceHintsAsync(tenantId, minConfidence: 0.8, limit: 1, CancellationToken.None);
|
||||
limited.Should().HaveCount(1);
|
||||
limited[0].Id.Should().Be(expectedOrder[0]);
|
||||
}
|
||||
|
||||
private IUnknownRepository GetRepository(string implementation)
|
||||
=> implementation switch
|
||||
{
|
||||
"postgres" => _repository,
|
||||
"efcore" => _efRepository,
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(implementation), implementation, null)
|
||||
};
|
||||
|
||||
private static IReadOnlyList<ProvenanceHint> CreateHints(DateTimeOffset generatedAt)
|
||||
{
|
||||
return
|
||||
[
|
||||
new ProvenanceHint
|
||||
{
|
||||
HintId = "hint:sha256:bbb222",
|
||||
Type = ProvenanceHintType.StringTableSignature,
|
||||
Confidence = 0.62,
|
||||
ConfidenceLevel = HintConfidence.Medium,
|
||||
Summary = "String table signature suggests distro package",
|
||||
Hypothesis = "Potential distro package from string table",
|
||||
Evidence = new ProvenanceEvidence
|
||||
{
|
||||
Raw = JsonDocument.Parse("""{"source":"strings"}""")
|
||||
},
|
||||
SuggestedActions =
|
||||
[
|
||||
new SuggestedAction
|
||||
{
|
||||
Action = "manual_triage",
|
||||
Priority = 2,
|
||||
Effort = "medium",
|
||||
Description = "Inspect package metadata manually"
|
||||
}
|
||||
],
|
||||
GeneratedAt = generatedAt,
|
||||
Source = "StringAnalyzer"
|
||||
},
|
||||
new ProvenanceHint
|
||||
{
|
||||
HintId = "hint:sha256:aaa111",
|
||||
Type = ProvenanceHintType.BuildIdMatch,
|
||||
Confidence = 0.91,
|
||||
ConfidenceLevel = HintConfidence.High,
|
||||
Summary = "Build-ID catalog match",
|
||||
Hypothesis = "Debian package candidate",
|
||||
Evidence = new ProvenanceEvidence
|
||||
{
|
||||
BuildId = new BuildIdEvidence
|
||||
{
|
||||
BuildId = "deadbeef",
|
||||
BuildIdType = "sha256",
|
||||
MatchedPackage = "openssl",
|
||||
MatchedVersion = "3.0.0",
|
||||
MatchedDistro = "debian",
|
||||
CatalogSource = "debian-security"
|
||||
}
|
||||
},
|
||||
SuggestedActions =
|
||||
[
|
||||
new SuggestedAction
|
||||
{
|
||||
Action = "verify_build_id",
|
||||
Priority = 1,
|
||||
Effort = "low",
|
||||
Description = "Cross-check Build-ID with distro metadata"
|
||||
}
|
||||
],
|
||||
GeneratedAt = generatedAt,
|
||||
Source = "BuildIdAnalyzer"
|
||||
}
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
|
||||
<PackageReference Include="NSubstitute" />
|
||||
<PackageReference Include="Npgsql" />
|
||||
<PackageReference Include="Testcontainers.PostgreSql" />
|
||||
<PackageReference Include="coverlet.collector">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Npgsql;
|
||||
using StellaOps.Unknowns.WebService.Endpoints;
|
||||
using Testcontainers.PostgreSql;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Unknowns.WebService.Tests;
|
||||
|
||||
public sealed class UnknownsEndpointsPersistenceTests : IAsyncLifetime
|
||||
{
|
||||
private readonly PostgreSqlContainer _postgres = new PostgreSqlBuilder()
|
||||
.WithImage("postgres:16")
|
||||
.Build();
|
||||
|
||||
private WebApplicationFactory<Program> _factory = null!;
|
||||
private HttpClient _client = null!;
|
||||
private string _connectionString = string.Empty;
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
await _postgres.StartAsync();
|
||||
_connectionString = _postgres.GetConnectionString();
|
||||
|
||||
await RunMigrationsAsync(_connectionString);
|
||||
|
||||
_factory = new WebApplicationFactory<Program>().WithWebHostBuilder(builder =>
|
||||
{
|
||||
builder.ConfigureAppConfiguration((_, config) =>
|
||||
{
|
||||
config.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Postgres:ConnectionString"] = _connectionString,
|
||||
["Postgres:SchemaName"] = "unknowns",
|
||||
["Authority:ResourceServer:Authority"] = "http://localhost",
|
||||
["Authority:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32",
|
||||
["Authority:ResourceServer:BypassNetworks:1"] = "::1/128"
|
||||
});
|
||||
});
|
||||
|
||||
builder.ConfigureTestServices(UnknownsTestSecurity.Configure);
|
||||
});
|
||||
|
||||
_client = _factory.CreateClient();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_client.Dispose();
|
||||
_factory.Dispose();
|
||||
await _postgres.DisposeAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", "Integration")]
|
||||
public async Task GetHighConfidenceHints_UsesPersistenceAndTenantFiltering()
|
||||
{
|
||||
var tenantId = "unknowns-persist-tenant";
|
||||
await SeedUnknownAsync(
|
||||
tenantId,
|
||||
Guid.Parse("11111111-1111-1111-1111-111111111111"),
|
||||
"pkg:npm/lodash@4.17.21",
|
||||
0.91m,
|
||||
"Debian package candidate",
|
||||
"verify_build_id");
|
||||
await SeedUnknownAsync(
|
||||
tenantId,
|
||||
Guid.Parse("22222222-2222-2222-2222-222222222222"),
|
||||
"pkg:npm/axios@0.27.2",
|
||||
0.61m,
|
||||
"Low confidence",
|
||||
"manual_triage");
|
||||
await SeedUnknownAsync(
|
||||
"other-tenant",
|
||||
Guid.Parse("33333333-3333-3333-3333-333333333333"),
|
||||
"pkg:npm/express@4.18.2",
|
||||
0.99m,
|
||||
"Other tenant",
|
||||
"manual_triage");
|
||||
|
||||
using var request = new HttpRequestMessage(
|
||||
HttpMethod.Get,
|
||||
"/api/unknowns/high-confidence?minConfidence=0.70&limit=10");
|
||||
request.Headers.Add("X-Tenant-Id", tenantId);
|
||||
|
||||
var response = await _client.SendAsync(request);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
var payload = await response.Content.ReadFromJsonAsync<UnknownsListResponse>();
|
||||
Assert.NotNull(payload);
|
||||
Assert.Single(payload.Items);
|
||||
|
||||
var item = payload.Items[0];
|
||||
Assert.Equal(tenantId, item.TenantId);
|
||||
Assert.Equal("pkg:npm/lodash@4.17.21", item.SubjectRef);
|
||||
Assert.Equal("Debian package candidate", item.BestHypothesis);
|
||||
Assert.Equal(0.91, item.CombinedConfidence!.Value, 3);
|
||||
Assert.NotEmpty(item.ProvenanceHints);
|
||||
}
|
||||
|
||||
private async Task SeedUnknownAsync(
|
||||
string tenantId,
|
||||
Guid id,
|
||||
string subjectRef,
|
||||
decimal combinedConfidence,
|
||||
string bestHypothesis,
|
||||
string suggestedAction)
|
||||
{
|
||||
var hintJson =
|
||||
"""
|
||||
[
|
||||
{
|
||||
"hint_id": "hint:sha256:seeded",
|
||||
"type": "BuildIdMatch",
|
||||
"confidence": 0.91,
|
||||
"confidence_level": "High",
|
||||
"summary": "Build-id matched package metadata",
|
||||
"hypothesis": "Debian package candidate",
|
||||
"evidence": {
|
||||
"build_id": {
|
||||
"build_id": "deadbeef",
|
||||
"build_id_type": "sha256",
|
||||
"matched_package": "openssl",
|
||||
"matched_version": "3.0.0"
|
||||
}
|
||||
},
|
||||
"suggested_actions": [
|
||||
{
|
||||
"action": "verify_build_id",
|
||||
"priority": 1,
|
||||
"effort": "low",
|
||||
"description": "Verify package provenance from distro metadata"
|
||||
}
|
||||
],
|
||||
"generated_at": "2026-03-01T10:00:00Z",
|
||||
"source": "integration-seed"
|
||||
}
|
||||
]
|
||||
""";
|
||||
|
||||
var subjectHash = id.ToString("N").PadRight(64, '0');
|
||||
|
||||
await using var dataSource = NpgsqlDataSource.Create(_connectionString);
|
||||
await using var connection = await dataSource.OpenConnectionAsync();
|
||||
await using var command = new NpgsqlCommand(
|
||||
"""
|
||||
INSERT INTO unknowns.unknown (
|
||||
id, tenant_id, subject_hash, subject_type, subject_ref,
|
||||
kind, severity, context, valid_from, sys_from,
|
||||
created_at, created_by, updated_at,
|
||||
provenance_hints, best_hypothesis, combined_confidence, primary_suggested_action
|
||||
) VALUES (
|
||||
@id, @tenantId, @subjectHash, 'package'::unknowns.subject_type, @subjectRef,
|
||||
'missing_feed'::unknowns.unknown_kind, 'high'::unknowns.unknown_severity, '{}'::jsonb, now(), now(),
|
||||
now(), 'integration-test', now(),
|
||||
@hints::jsonb, @bestHypothesis, @combinedConfidence, @primarySuggestedAction
|
||||
);
|
||||
""",
|
||||
connection);
|
||||
|
||||
command.Parameters.AddWithValue("id", id);
|
||||
command.Parameters.AddWithValue("tenantId", tenantId);
|
||||
command.Parameters.AddWithValue("subjectHash", subjectHash);
|
||||
command.Parameters.AddWithValue("subjectRef", subjectRef);
|
||||
command.Parameters.AddWithValue("hints", hintJson);
|
||||
command.Parameters.AddWithValue("bestHypothesis", bestHypothesis);
|
||||
command.Parameters.AddWithValue("combinedConfidence", combinedConfidence);
|
||||
command.Parameters.AddWithValue("primarySuggestedAction", suggestedAction);
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
private static async Task RunMigrationsAsync(string connectionString)
|
||||
{
|
||||
await using var rawDataSource = NpgsqlDataSource.Create(connectionString);
|
||||
await using var connection = await rawDataSource.OpenConnectionAsync();
|
||||
|
||||
const string schema = """
|
||||
CREATE SCHEMA IF NOT EXISTS unknowns;
|
||||
CREATE SCHEMA IF NOT EXISTS unknowns_app;
|
||||
|
||||
CREATE OR REPLACE FUNCTION unknowns_app.require_current_tenant()
|
||||
RETURNS TEXT
|
||||
LANGUAGE plpgsql STABLE SECURITY DEFINER
|
||||
AS $$
|
||||
DECLARE
|
||||
v_tenant TEXT;
|
||||
BEGIN
|
||||
v_tenant := current_setting('app.tenant_id', true);
|
||||
IF v_tenant IS NULL OR v_tenant = '' THEN
|
||||
RAISE EXCEPTION 'app.tenant_id session variable not set';
|
||||
END IF;
|
||||
RETURN v_tenant;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DO $$ BEGIN
|
||||
CREATE TYPE unknowns.subject_type AS ENUM (
|
||||
'package', 'ecosystem', 'version', 'sbom_edge', 'file', 'runtime'
|
||||
);
|
||||
EXCEPTION WHEN duplicate_object THEN null;
|
||||
END $$;
|
||||
|
||||
DO $$ BEGIN
|
||||
CREATE TYPE unknowns.unknown_kind AS ENUM (
|
||||
'missing_sbom', 'ambiguous_package', 'missing_feed', 'unresolved_edge',
|
||||
'no_version_info', 'unknown_ecosystem', 'partial_match',
|
||||
'version_range_unbounded', 'unsupported_format', 'transitive_gap'
|
||||
);
|
||||
EXCEPTION WHEN duplicate_object THEN null;
|
||||
END $$;
|
||||
|
||||
DO $$ BEGIN
|
||||
CREATE TYPE unknowns.unknown_severity AS ENUM (
|
||||
'critical', 'high', 'medium', 'low', 'info'
|
||||
);
|
||||
EXCEPTION WHEN duplicate_object THEN null;
|
||||
END $$;
|
||||
|
||||
DO $$ BEGIN
|
||||
CREATE TYPE unknowns.resolution_type AS ENUM (
|
||||
'feed_updated', 'sbom_provided', 'manual_mapping',
|
||||
'superseded', 'false_positive', 'wont_fix'
|
||||
);
|
||||
EXCEPTION WHEN duplicate_object THEN null;
|
||||
END $$;
|
||||
|
||||
DO $$ BEGIN
|
||||
CREATE TYPE unknowns.triage_band AS ENUM ('hot', 'warm', 'cold');
|
||||
EXCEPTION WHEN duplicate_object THEN null;
|
||||
END $$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS unknowns.unknown (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tenant_id TEXT NOT NULL,
|
||||
subject_hash CHAR(64) NOT NULL,
|
||||
subject_type unknowns.subject_type NOT NULL,
|
||||
subject_ref TEXT NOT NULL,
|
||||
kind unknowns.unknown_kind NOT NULL,
|
||||
severity unknowns.unknown_severity,
|
||||
context JSONB NOT NULL DEFAULT '{}',
|
||||
source_scan_id UUID,
|
||||
source_graph_id UUID,
|
||||
source_sbom_digest TEXT,
|
||||
valid_from TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
valid_to TIMESTAMPTZ,
|
||||
sys_from TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
sys_to TIMESTAMPTZ,
|
||||
resolved_at TIMESTAMPTZ,
|
||||
resolution_type unknowns.resolution_type,
|
||||
resolution_ref TEXT,
|
||||
resolution_notes TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
created_by TEXT NOT NULL DEFAULT 'system',
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
popularity_score FLOAT DEFAULT 0.0,
|
||||
deployment_count INT DEFAULT 0,
|
||||
exploit_potential_score FLOAT DEFAULT 0.0,
|
||||
uncertainty_score FLOAT DEFAULT 0.0,
|
||||
uncertainty_flags JSONB DEFAULT '{}'::jsonb,
|
||||
centrality_score FLOAT DEFAULT 0.0,
|
||||
degree_centrality INT DEFAULT 0,
|
||||
betweenness_centrality FLOAT DEFAULT 0.0,
|
||||
staleness_score FLOAT DEFAULT 0.0,
|
||||
days_since_analysis INT DEFAULT 0,
|
||||
composite_score FLOAT DEFAULT 0.0,
|
||||
triage_band unknowns.triage_band DEFAULT 'cold',
|
||||
scoring_trace JSONB,
|
||||
rescan_attempts INT DEFAULT 0,
|
||||
last_rescan_result TEXT,
|
||||
next_scheduled_rescan TIMESTAMPTZ,
|
||||
last_analyzed_at TIMESTAMPTZ,
|
||||
evidence_set_hash BYTEA,
|
||||
graph_slice_hash BYTEA,
|
||||
provenance_hints JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
best_hypothesis TEXT,
|
||||
combined_confidence NUMERIC(4,4) CHECK (combined_confidence IS NULL OR (combined_confidence >= 0 AND combined_confidence <= 1)),
|
||||
primary_suggested_action TEXT
|
||||
);
|
||||
""";
|
||||
|
||||
await using var command = new NpgsqlCommand(schema, connection);
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Unknowns.Core.Models;
|
||||
@@ -38,7 +39,10 @@ public sealed class UnknownsEndpointsTests : IClassFixture<WebApplicationFactory
|
||||
var settings = new Dictionary<string, string?>
|
||||
{
|
||||
["ConnectionStrings:UnknownsDb"] =
|
||||
"Host=localhost;Database=unknowns_test;Username=test;Password=test"
|
||||
"Host=localhost;Database=unknowns_test;Username=test;Password=test",
|
||||
["Authority:ResourceServer:Authority"] = "http://localhost",
|
||||
["Authority:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32",
|
||||
["Authority:ResourceServer:BypassNetworks:1"] = "::1/128"
|
||||
};
|
||||
config.AddInMemoryCollection(settings);
|
||||
});
|
||||
@@ -56,6 +60,8 @@ public sealed class UnknownsEndpointsTests : IClassFixture<WebApplicationFactory
|
||||
// Add mock repository
|
||||
services.AddSingleton(_mockRepository);
|
||||
});
|
||||
|
||||
builder.ConfigureTestServices(UnknownsTestSecurity.Configure);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
using System.Security.Claims;
|
||||
using System.Text.Encodings.Web;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Unknowns.WebService.Tests;
|
||||
|
||||
internal static class UnknownsTestSecurity
|
||||
{
|
||||
public static void Configure(IServiceCollection services)
|
||||
{
|
||||
services.AddAuthentication(TestAuthHandler.SchemeName)
|
||||
.AddScheme<AuthenticationSchemeOptions, TestAuthHandler>(
|
||||
TestAuthHandler.SchemeName,
|
||||
_ => { });
|
||||
|
||||
services.PostConfigureAll<AuthenticationOptions>(options =>
|
||||
{
|
||||
options.DefaultAuthenticateScheme = TestAuthHandler.SchemeName;
|
||||
options.DefaultChallengeScheme = TestAuthHandler.SchemeName;
|
||||
options.DefaultScheme = TestAuthHandler.SchemeName;
|
||||
});
|
||||
|
||||
services.RemoveAll<IAuthorizationHandler>();
|
||||
services.AddSingleton<IAuthorizationHandler, AllowAllAuthorizationHandler>();
|
||||
}
|
||||
|
||||
private sealed class TestAuthHandler : AuthenticationHandler<AuthenticationSchemeOptions>
|
||||
{
|
||||
public const string SchemeName = "UnknownsTestScheme";
|
||||
|
||||
public TestAuthHandler(
|
||||
IOptionsMonitor<AuthenticationSchemeOptions> options,
|
||||
ILoggerFactory logger,
|
||||
UrlEncoder encoder)
|
||||
: base(options, logger, encoder)
|
||||
{
|
||||
}
|
||||
|
||||
protected override Task<AuthenticateResult> HandleAuthenticateAsync()
|
||||
{
|
||||
var claims = new[]
|
||||
{
|
||||
new Claim("scope", "unknowns.read unknowns.write"),
|
||||
new Claim("scp", "unknowns.read unknowns.write")
|
||||
};
|
||||
|
||||
var principal = new ClaimsPrincipal(new ClaimsIdentity(claims, SchemeName));
|
||||
var ticket = new AuthenticationTicket(principal, SchemeName);
|
||||
return Task.FromResult(AuthenticateResult.Success(ticket));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class AllowAllAuthorizationHandler : IAuthorizationHandler
|
||||
{
|
||||
public Task HandleAsync(AuthorizationHandlerContext context)
|
||||
{
|
||||
foreach (var requirement in context.PendingRequirements.ToList())
|
||||
{
|
||||
context.Succeed(requirement);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user