Close scratch iteration 009 grouped policy and VEX audit repairs
This commit is contained in:
@@ -6,6 +6,7 @@ using StellaOps.VexHub.Core.Export;
|
||||
using StellaOps.VexHub.Core.Models;
|
||||
using StellaOps.VexHub.WebService.Models;
|
||||
using StellaOps.VexHub.WebService.Security;
|
||||
using StellaOps.VexLens.Models;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
@@ -86,7 +87,7 @@ public static class VexHubEndpointExtensions
|
||||
string cveId,
|
||||
[FromQuery] int? limit,
|
||||
[FromQuery] int? offset,
|
||||
IVexStatementRepository repository,
|
||||
[FromServices] IVexStatementRepository repository,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var statements = await repository.GetByCveAsync(cveId, limit ?? 100, offset ?? 0, cancellationToken);
|
||||
@@ -107,7 +108,7 @@ public static class VexHubEndpointExtensions
|
||||
string purl,
|
||||
[FromQuery] int? limit,
|
||||
[FromQuery] int? offset,
|
||||
IVexStatementRepository repository,
|
||||
[FromServices] IVexStatementRepository repository,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// URL decode the PURL
|
||||
@@ -130,7 +131,7 @@ public static class VexHubEndpointExtensions
|
||||
string sourceId,
|
||||
[FromQuery] int? limit,
|
||||
[FromQuery] int? offset,
|
||||
IVexStatementRepository repository,
|
||||
[FromServices] IVexStatementRepository repository,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var statements = await repository.GetBySourceAsync(sourceId, limit ?? 100, offset ?? 0, cancellationToken);
|
||||
@@ -149,7 +150,7 @@ public static class VexHubEndpointExtensions
|
||||
|
||||
private static async Task<IResult> GetById(
|
||||
Guid id,
|
||||
IVexStatementRepository repository,
|
||||
[FromServices] IVexStatementRepository repository,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var statement = await repository.GetByIdAsync(id, cancellationToken);
|
||||
@@ -168,7 +169,7 @@ public static class VexHubEndpointExtensions
|
||||
[FromQuery] bool? isFlagged,
|
||||
[FromQuery] int? limit,
|
||||
[FromQuery] int? offset,
|
||||
IVexStatementRepository repository,
|
||||
[FromServices] IVexStatementRepository repository,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var filter = new VexStatementFilter
|
||||
@@ -192,7 +193,9 @@ public static class VexHubEndpointExtensions
|
||||
}
|
||||
|
||||
private static async Task<IResult> GetStats(
|
||||
IVexStatementRepository repository,
|
||||
[FromServices] IVexSourceRepository sourceRepository,
|
||||
[FromServices] IVexStatementRepository repository,
|
||||
[FromServices] TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var totalCount = await repository.GetCountAsync(cancellationToken: cancellationToken);
|
||||
@@ -202,22 +205,123 @@ public static class VexHubEndpointExtensions
|
||||
var flaggedCount = await repository.GetCountAsync(
|
||||
new VexStatementFilter { IsFlagged = true },
|
||||
cancellationToken);
|
||||
var allSources = await sourceRepository.GetAllAsync(cancellationToken);
|
||||
var recentStatements = await repository.SearchAsync(
|
||||
new VexStatementFilter(),
|
||||
limit: 10_000,
|
||||
offset: null,
|
||||
cancellationToken);
|
||||
|
||||
var byStatus = recentStatements
|
||||
.GroupBy(statement => FormatStatusKey(statement.Status), StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(group => group.Key, group => (long)group.Count(), StringComparer.OrdinalIgnoreCase);
|
||||
EnsureStatusBuckets(byStatus);
|
||||
|
||||
var bySource = recentStatements
|
||||
.GroupBy(statement => ResolveSourceBucket(statement.SourceId, allSources), StringComparer.OrdinalIgnoreCase)
|
||||
.OrderByDescending(group => group.Count())
|
||||
.ThenBy(group => group.Key, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(group => group.Key, group => (long)group.Count(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var recentActivity = recentStatements
|
||||
.OrderByDescending(statement => statement.SourceUpdatedAt ?? statement.UpdatedAt ?? statement.IngestedAt)
|
||||
.ThenBy(statement => statement.SourceId, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(20)
|
||||
.Select(statement => new VexHubActivityItem
|
||||
{
|
||||
StatementId = statement.Id,
|
||||
CveId = statement.VulnerabilityId,
|
||||
Action = ResolveActivityAction(statement),
|
||||
Timestamp = statement.SourceUpdatedAt ?? statement.UpdatedAt ?? statement.IngestedAt
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
var trendWindowStart = timeProvider.GetUtcNow().UtcDateTime.Date.AddDays(-6);
|
||||
var trends = Enumerable.Range(0, 7)
|
||||
.Select(offset => trendWindowStart.AddDays(offset))
|
||||
.Select(day =>
|
||||
{
|
||||
var dayStatements = recentStatements
|
||||
.Where(statement => (statement.SourceUpdatedAt ?? statement.UpdatedAt ?? statement.IngestedAt).UtcDateTime.Date == day)
|
||||
.ToArray();
|
||||
|
||||
return new VexHubTrendPoint
|
||||
{
|
||||
Date = DateOnly.FromDateTime(day),
|
||||
Affected = dayStatements.Count(statement => statement.Status == VexStatus.Affected),
|
||||
NotAffected = dayStatements.Count(statement => statement.Status == VexStatus.NotAffected),
|
||||
Fixed = dayStatements.Count(statement => statement.Status == VexStatus.Fixed),
|
||||
Investigating = dayStatements.Count(statement => statement.Status == VexStatus.UnderInvestigation)
|
||||
};
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
return Results.Ok(new VexHubStats
|
||||
{
|
||||
TotalStatements = totalCount,
|
||||
VerifiedStatements = verifiedCount,
|
||||
FlaggedStatements = flaggedCount,
|
||||
GeneratedAt = DateTimeOffset.UtcNow
|
||||
ByStatus = byStatus,
|
||||
BySource = bySource,
|
||||
RecentActivity = recentActivity,
|
||||
Trends = trends,
|
||||
GeneratedAt = timeProvider.GetUtcNow()
|
||||
});
|
||||
}
|
||||
|
||||
private static IResult GetIndex()
|
||||
private static void EnsureStatusBuckets(IDictionary<string, long> byStatus)
|
||||
{
|
||||
foreach (var key in new[] { "affected", "not_affected", "fixed", "under_investigation" })
|
||||
{
|
||||
if (!byStatus.ContainsKey(key))
|
||||
{
|
||||
byStatus[key] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string ResolveSourceBucket(string sourceId, IReadOnlyList<VexSource> allSources)
|
||||
{
|
||||
var source = allSources.FirstOrDefault(candidate =>
|
||||
string.Equals(candidate.SourceId, sourceId, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
return source?.IssuerCategory switch
|
||||
{
|
||||
IssuerCategory.Vendor => "vendor",
|
||||
IssuerCategory.Community => "community",
|
||||
IssuerCategory.Distributor => "distributor",
|
||||
IssuerCategory.Internal => "internal",
|
||||
IssuerCategory.Aggregator => "aggregator",
|
||||
_ => string.IsNullOrWhiteSpace(sourceId) ? "unknown" : sourceId
|
||||
};
|
||||
}
|
||||
|
||||
private static string ResolveActivityAction(AggregatedVexStatement statement)
|
||||
{
|
||||
var updatedAt = statement.SourceUpdatedAt ?? statement.UpdatedAt;
|
||||
if (updatedAt.HasValue && updatedAt.Value > statement.IngestedAt.AddMinutes(1))
|
||||
{
|
||||
return "updated";
|
||||
}
|
||||
|
||||
return statement.IsFlagged ? "superseded" : "created";
|
||||
}
|
||||
|
||||
private static string FormatStatusKey(VexStatus status) => status switch
|
||||
{
|
||||
VexStatus.NotAffected => "not_affected",
|
||||
VexStatus.Affected => "affected",
|
||||
VexStatus.Fixed => "fixed",
|
||||
VexStatus.UnderInvestigation => "under_investigation",
|
||||
_ => status.ToString().ToLowerInvariant()
|
||||
};
|
||||
|
||||
private static IResult GetIndex([FromServices] TimeProvider timeProvider)
|
||||
{
|
||||
return Results.Ok(new VexIndexManifest
|
||||
{
|
||||
Version = "1.0",
|
||||
LastUpdated = DateTimeOffset.UtcNow,
|
||||
LastUpdated = timeProvider.GetUtcNow(),
|
||||
Endpoints = new VexIndexEndpoints
|
||||
{
|
||||
ByCve = "/api/v1/vex/cve/{cve}",
|
||||
|
||||
@@ -32,9 +32,30 @@ public sealed class VexHubStats
|
||||
public required long TotalStatements { get; init; }
|
||||
public required long VerifiedStatements { get; init; }
|
||||
public required long FlaggedStatements { get; init; }
|
||||
public required IReadOnlyDictionary<string, long> ByStatus { get; init; }
|
||||
public required IReadOnlyDictionary<string, long> BySource { get; init; }
|
||||
public required IReadOnlyList<VexHubActivityItem> RecentActivity { get; init; }
|
||||
public required IReadOnlyList<VexHubTrendPoint> Trends { get; init; }
|
||||
public required DateTimeOffset GeneratedAt { get; init; }
|
||||
}
|
||||
|
||||
public sealed class VexHubActivityItem
|
||||
{
|
||||
public required Guid StatementId { get; init; }
|
||||
public required string CveId { get; init; }
|
||||
public required string Action { get; init; }
|
||||
public required DateTimeOffset Timestamp { get; init; }
|
||||
}
|
||||
|
||||
public sealed class VexHubTrendPoint
|
||||
{
|
||||
public required DateOnly Date { get; init; }
|
||||
public required int Affected { get; init; }
|
||||
public required int NotAffected { get; init; }
|
||||
public required int Fixed { get; init; }
|
||||
public required int Investigating { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// VEX Hub index manifest for tool integration.
|
||||
/// </summary>
|
||||
|
||||
@@ -121,10 +121,9 @@ public partial class VexHubDbContext : DbContext
|
||||
entity.Property(e => e.UpdatedAt).HasColumnName("updated_at");
|
||||
entity.Property(e => e.ContentDigest).HasColumnName("content_digest");
|
||||
|
||||
// search_vector is maintained by a DB trigger; EF should not write to it
|
||||
entity.Property(e => e.SearchVector)
|
||||
.HasColumnType("tsvector")
|
||||
.HasColumnName("search_vector");
|
||||
// search_vector is maintained by a DB trigger and is not used by the EF repositories.
|
||||
// Keeping it as a mapped string property breaks runtime model validation for Npgsql.
|
||||
entity.Ignore(e => e.SearchVector);
|
||||
});
|
||||
|
||||
// ── conflicts ────────────────────────────────────────────────────
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Infrastructure.Postgres.Migrations;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
using StellaOps.VexHub.Core;
|
||||
using StellaOps.VexHub.Persistence.Postgres;
|
||||
@@ -22,6 +23,13 @@ public static class VexHubPersistenceExtensions
|
||||
services.Configure<PostgresOptions>(configuration.GetSection("Postgres"));
|
||||
|
||||
services.AddSingleton<VexHubDataSource>();
|
||||
services.AddStartupMigrations(
|
||||
VexHubDataSource.DefaultSchemaName,
|
||||
"VexHub.Persistence",
|
||||
typeof(VexHubDataSource).Assembly);
|
||||
services.AddScoped<IVexSourceRepository, PostgresVexSourceRepository>();
|
||||
services.AddScoped<IVexConflictRepository, PostgresVexConflictRepository>();
|
||||
services.AddScoped<IVexIngestionJobRepository, PostgresVexIngestionJobRepository>();
|
||||
services.AddScoped<IVexStatementRepository, PostgresVexStatementRepository>();
|
||||
services.AddScoped<IVexProvenanceRepository, PostgresVexProvenanceRepository>();
|
||||
|
||||
@@ -38,6 +46,13 @@ public static class VexHubPersistenceExtensions
|
||||
services.Configure(configureOptions);
|
||||
|
||||
services.AddSingleton<VexHubDataSource>();
|
||||
services.AddStartupMigrations(
|
||||
VexHubDataSource.DefaultSchemaName,
|
||||
"VexHub.Persistence",
|
||||
typeof(VexHubDataSource).Assembly);
|
||||
services.AddScoped<IVexSourceRepository, PostgresVexSourceRepository>();
|
||||
services.AddScoped<IVexConflictRepository, PostgresVexConflictRepository>();
|
||||
services.AddScoped<IVexIngestionJobRepository, PostgresVexIngestionJobRepository>();
|
||||
services.AddScoped<IVexStatementRepository, PostgresVexStatementRepository>();
|
||||
services.AddScoped<IVexProvenanceRepository, PostgresVexProvenanceRepository>();
|
||||
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using StellaOps.VexHub.Core;
|
||||
using StellaOps.VexHub.Core.Models;
|
||||
|
||||
namespace StellaOps.VexHub.Persistence.Postgres.Repositories;
|
||||
|
||||
using EfModels = StellaOps.VexHub.Persistence.EfCore.Models;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL (EF Core) implementation of the VEX conflict repository.
|
||||
/// </summary>
|
||||
public sealed class PostgresVexConflictRepository : IVexConflictRepository
|
||||
{
|
||||
private const int CommandTimeoutSeconds = 30;
|
||||
|
||||
private readonly VexHubDataSource _dataSource;
|
||||
|
||||
public PostgresVexConflictRepository(VexHubDataSource dataSource)
|
||||
{
|
||||
_dataSource = dataSource;
|
||||
}
|
||||
|
||||
public async Task<VexConflict> AddAsync(VexConflict conflict, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = ToEntity(conflict);
|
||||
dbContext.Conflicts.Add(entity);
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
|
||||
return ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<VexConflict?> GetByIdAsync(Guid id, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.Conflicts
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(conflict => conflict.Id == id, cancellationToken);
|
||||
|
||||
return entity is null ? null : ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<VexConflict>> GetByVulnerabilityProductAsync(
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entities = await dbContext.Conflicts
|
||||
.AsNoTracking()
|
||||
.Where(conflict => conflict.VulnerabilityId == vulnerabilityId && conflict.ProductKey == productKey)
|
||||
.OrderByDescending(conflict => conflict.DetectedAt)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return entities.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<VexConflict>> GetOpenConflictsAsync(
|
||||
int? limit = null,
|
||||
int? offset = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
IQueryable<EfModels.VexConflict> query = dbContext.Conflicts
|
||||
.AsNoTracking()
|
||||
.Where(conflict => conflict.ResolutionStatus == "open")
|
||||
.OrderByDescending(conflict => conflict.DetectedAt);
|
||||
|
||||
if (offset.HasValue)
|
||||
{
|
||||
query = query.Skip(offset.Value);
|
||||
}
|
||||
|
||||
if (limit.HasValue)
|
||||
{
|
||||
query = query.Take(limit.Value);
|
||||
}
|
||||
|
||||
var entities = await query.ToListAsync(cancellationToken);
|
||||
return entities.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<VexConflict>> GetBySeverityAsync(
|
||||
ConflictSeverity severity,
|
||||
ConflictResolutionStatus? status = null,
|
||||
int? limit = null,
|
||||
int? offset = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
IQueryable<EfModels.VexConflict> query = dbContext.Conflicts
|
||||
.AsNoTracking()
|
||||
.Where(conflict => conflict.Severity == FormatSeverity(severity))
|
||||
.OrderByDescending(conflict => conflict.DetectedAt);
|
||||
|
||||
if (status.HasValue)
|
||||
{
|
||||
var resolutionStatus = FormatResolutionStatus(status.Value);
|
||||
query = query.Where(conflict => conflict.ResolutionStatus == resolutionStatus);
|
||||
}
|
||||
|
||||
if (offset.HasValue)
|
||||
{
|
||||
query = query.Skip(offset.Value);
|
||||
}
|
||||
|
||||
if (limit.HasValue)
|
||||
{
|
||||
query = query.Take(limit.Value);
|
||||
}
|
||||
|
||||
var entities = await query.ToListAsync(cancellationToken);
|
||||
return entities.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async Task ResolveAsync(
|
||||
Guid id,
|
||||
ConflictResolutionStatus status,
|
||||
string? resolutionMethod,
|
||||
Guid? winningStatementId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.Conflicts.FirstOrDefaultAsync(conflict => conflict.Id == id, cancellationToken);
|
||||
if (entity is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
entity.ResolutionStatus = FormatResolutionStatus(status);
|
||||
entity.ResolutionMethod = resolutionMethod;
|
||||
entity.WinningStatementId = winningStatementId;
|
||||
entity.ResolvedAt = DateTime.UtcNow;
|
||||
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<long> GetOpenConflictCountAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
return await dbContext.Conflicts
|
||||
.AsNoTracking()
|
||||
.LongCountAsync(conflict => conflict.ResolutionStatus == "open", cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyDictionary<ConflictSeverity, long>> GetConflictCountsBySeverityAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var grouped = await dbContext.Conflicts
|
||||
.AsNoTracking()
|
||||
.GroupBy(conflict => conflict.Severity)
|
||||
.Select(group => new { Severity = group.Key, Count = group.LongCount() })
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return grouped.ToDictionary(item => ParseSeverity(item.Severity), item => item.Count);
|
||||
}
|
||||
|
||||
private static EfModels.VexConflict ToEntity(VexConflict model) => new()
|
||||
{
|
||||
Id = model.Id,
|
||||
VulnerabilityId = model.VulnerabilityId,
|
||||
ProductKey = model.ProductKey,
|
||||
ConflictingStatementIds = model.ConflictingStatementIds.ToArray(),
|
||||
Severity = FormatSeverity(model.Severity),
|
||||
Description = model.Description,
|
||||
ResolutionStatus = FormatResolutionStatus(model.ResolutionStatus),
|
||||
ResolutionMethod = model.ResolutionMethod,
|
||||
WinningStatementId = model.WinningStatementId,
|
||||
DetectedAt = model.DetectedAt.UtcDateTime,
|
||||
ResolvedAt = model.ResolvedAt?.UtcDateTime
|
||||
};
|
||||
|
||||
private static VexConflict ToModel(EfModels.VexConflict entity) => new()
|
||||
{
|
||||
Id = entity.Id,
|
||||
VulnerabilityId = entity.VulnerabilityId,
|
||||
ProductKey = entity.ProductKey,
|
||||
ConflictingStatementIds = entity.ConflictingStatementIds,
|
||||
Severity = ParseSeverity(entity.Severity),
|
||||
Description = entity.Description,
|
||||
ResolutionStatus = ParseResolutionStatus(entity.ResolutionStatus),
|
||||
ResolutionMethod = entity.ResolutionMethod,
|
||||
WinningStatementId = entity.WinningStatementId,
|
||||
DetectedAt = new DateTimeOffset(entity.DetectedAt, TimeSpan.Zero),
|
||||
ResolvedAt = entity.ResolvedAt.HasValue ? new DateTimeOffset(entity.ResolvedAt.Value, TimeSpan.Zero) : null
|
||||
};
|
||||
|
||||
private static string FormatSeverity(ConflictSeverity severity) => severity switch
|
||||
{
|
||||
ConflictSeverity.Low => "low",
|
||||
ConflictSeverity.Medium => "medium",
|
||||
ConflictSeverity.High => "high",
|
||||
_ => "critical"
|
||||
};
|
||||
|
||||
private static ConflictSeverity ParseSeverity(string severity) => severity.ToLowerInvariant() switch
|
||||
{
|
||||
"low" => ConflictSeverity.Low,
|
||||
"medium" => ConflictSeverity.Medium,
|
||||
"high" => ConflictSeverity.High,
|
||||
_ => ConflictSeverity.Critical
|
||||
};
|
||||
|
||||
private static string FormatResolutionStatus(ConflictResolutionStatus status) => status switch
|
||||
{
|
||||
ConflictResolutionStatus.Open => "open",
|
||||
ConflictResolutionStatus.AutoResolved => "auto_resolved",
|
||||
ConflictResolutionStatus.ManuallyResolved => "manually_resolved",
|
||||
_ => "suppressed"
|
||||
};
|
||||
|
||||
private static ConflictResolutionStatus ParseResolutionStatus(string status) => status.ToLowerInvariant() switch
|
||||
{
|
||||
"open" => ConflictResolutionStatus.Open,
|
||||
"auto_resolved" => ConflictResolutionStatus.AutoResolved,
|
||||
"manually_resolved" => ConflictResolutionStatus.ManuallyResolved,
|
||||
_ => ConflictResolutionStatus.Suppressed
|
||||
};
|
||||
|
||||
private static string GetSchemaName() => VexHubDataSource.DefaultSchemaName;
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using StellaOps.VexHub.Core;
|
||||
using StellaOps.VexHub.Core.Models;
|
||||
|
||||
namespace StellaOps.VexHub.Persistence.Postgres.Repositories;
|
||||
|
||||
using EfModels = StellaOps.VexHub.Persistence.EfCore.Models;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL (EF Core) implementation of the VEX ingestion job repository.
|
||||
/// </summary>
|
||||
public sealed class PostgresVexIngestionJobRepository : IVexIngestionJobRepository
|
||||
{
|
||||
private const int CommandTimeoutSeconds = 30;
|
||||
|
||||
private readonly VexHubDataSource _dataSource;
|
||||
|
||||
public PostgresVexIngestionJobRepository(VexHubDataSource dataSource)
|
||||
{
|
||||
_dataSource = dataSource;
|
||||
}
|
||||
|
||||
public async Task<VexIngestionJob> CreateAsync(VexIngestionJob job, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = ToEntity(job);
|
||||
dbContext.IngestionJobs.Add(entity);
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
|
||||
return ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<VexIngestionJob> UpdateAsync(VexIngestionJob job, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.IngestionJobs.FirstOrDefaultAsync(existing => existing.JobId == job.JobId, cancellationToken);
|
||||
if (entity is null)
|
||||
{
|
||||
entity = ToEntity(job);
|
||||
dbContext.IngestionJobs.Add(entity);
|
||||
}
|
||||
else
|
||||
{
|
||||
Apply(job, entity);
|
||||
}
|
||||
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
return ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<VexIngestionJob?> GetByIdAsync(Guid jobId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.IngestionJobs
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(job => job.JobId == jobId, cancellationToken);
|
||||
|
||||
return entity is null ? null : ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<VexIngestionJob?> GetLatestBySourceAsync(string sourceId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.IngestionJobs
|
||||
.AsNoTracking()
|
||||
.Where(job => job.SourceId == sourceId)
|
||||
.OrderByDescending(job => job.StartedAt)
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
|
||||
return entity is null ? null : ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<VexIngestionJob>> GetByStatusAsync(
|
||||
IngestionJobStatus status,
|
||||
int? limit = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
IQueryable<EfModels.VexIngestionJob> query = dbContext.IngestionJobs
|
||||
.AsNoTracking()
|
||||
.Where(job => job.Status == FormatStatus(status))
|
||||
.OrderByDescending(job => job.StartedAt);
|
||||
|
||||
if (limit.HasValue)
|
||||
{
|
||||
query = query.Take(limit.Value);
|
||||
}
|
||||
|
||||
var entities = await query.ToListAsync(cancellationToken);
|
||||
return entities.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<VexIngestionJob>> GetRunningJobsAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
return GetByStatusAsync(IngestionJobStatus.Running, null, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task UpdateProgressAsync(
|
||||
Guid jobId,
|
||||
int documentsProcessed,
|
||||
int statementsIngested,
|
||||
int statementsDeduplicated,
|
||||
int conflictsDetected,
|
||||
string? checkpoint = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.IngestionJobs.FirstOrDefaultAsync(job => job.JobId == jobId, cancellationToken);
|
||||
if (entity is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
entity.DocumentsProcessed = documentsProcessed;
|
||||
entity.StatementsIngested = statementsIngested;
|
||||
entity.StatementsDeduplicated = statementsDeduplicated;
|
||||
entity.ConflictsDetected = conflictsDetected;
|
||||
entity.Checkpoint = checkpoint;
|
||||
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task CompleteAsync(
|
||||
Guid jobId,
|
||||
int documentsProcessed,
|
||||
int statementsIngested,
|
||||
int statementsDeduplicated,
|
||||
int conflictsDetected,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.IngestionJobs.FirstOrDefaultAsync(job => job.JobId == jobId, cancellationToken);
|
||||
if (entity is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
entity.Status = FormatStatus(IngestionJobStatus.Completed);
|
||||
entity.DocumentsProcessed = documentsProcessed;
|
||||
entity.StatementsIngested = statementsIngested;
|
||||
entity.StatementsDeduplicated = statementsDeduplicated;
|
||||
entity.ConflictsDetected = conflictsDetected;
|
||||
entity.CompletedAt = DateTime.UtcNow;
|
||||
entity.ErrorMessage = null;
|
||||
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task FailAsync(Guid jobId, string errorMessage, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.IngestionJobs.FirstOrDefaultAsync(job => job.JobId == jobId, cancellationToken);
|
||||
if (entity is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
entity.Status = FormatStatus(IngestionJobStatus.Failed);
|
||||
entity.ErrorMessage = errorMessage;
|
||||
entity.ErrorCount += 1;
|
||||
entity.CompletedAt = DateTime.UtcNow;
|
||||
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private static EfModels.VexIngestionJob ToEntity(VexIngestionJob model) => new()
|
||||
{
|
||||
JobId = model.JobId,
|
||||
SourceId = model.SourceId,
|
||||
Status = FormatStatus(model.Status),
|
||||
StartedAt = model.StartedAt.UtcDateTime,
|
||||
CompletedAt = model.CompletedAt?.UtcDateTime,
|
||||
DocumentsProcessed = model.DocumentsProcessed,
|
||||
StatementsIngested = model.StatementsIngested,
|
||||
StatementsDeduplicated = model.StatementsDeduplicated,
|
||||
ConflictsDetected = model.ConflictsDetected,
|
||||
ErrorCount = model.ErrorCount,
|
||||
ErrorMessage = model.ErrorMessage,
|
||||
Checkpoint = model.Checkpoint
|
||||
};
|
||||
|
||||
private static void Apply(VexIngestionJob model, EfModels.VexIngestionJob entity)
|
||||
{
|
||||
entity.SourceId = model.SourceId;
|
||||
entity.Status = FormatStatus(model.Status);
|
||||
entity.StartedAt = model.StartedAt.UtcDateTime;
|
||||
entity.CompletedAt = model.CompletedAt?.UtcDateTime;
|
||||
entity.DocumentsProcessed = model.DocumentsProcessed;
|
||||
entity.StatementsIngested = model.StatementsIngested;
|
||||
entity.StatementsDeduplicated = model.StatementsDeduplicated;
|
||||
entity.ConflictsDetected = model.ConflictsDetected;
|
||||
entity.ErrorCount = model.ErrorCount;
|
||||
entity.ErrorMessage = model.ErrorMessage;
|
||||
entity.Checkpoint = model.Checkpoint;
|
||||
}
|
||||
|
||||
private static VexIngestionJob ToModel(EfModels.VexIngestionJob entity) => new()
|
||||
{
|
||||
JobId = entity.JobId,
|
||||
SourceId = entity.SourceId,
|
||||
Status = ParseStatus(entity.Status),
|
||||
StartedAt = new DateTimeOffset(entity.StartedAt, TimeSpan.Zero),
|
||||
CompletedAt = entity.CompletedAt.HasValue ? new DateTimeOffset(entity.CompletedAt.Value, TimeSpan.Zero) : null,
|
||||
DocumentsProcessed = entity.DocumentsProcessed,
|
||||
StatementsIngested = entity.StatementsIngested,
|
||||
StatementsDeduplicated = entity.StatementsDeduplicated,
|
||||
ConflictsDetected = entity.ConflictsDetected,
|
||||
ErrorCount = entity.ErrorCount,
|
||||
ErrorMessage = entity.ErrorMessage,
|
||||
Checkpoint = entity.Checkpoint
|
||||
};
|
||||
|
||||
private static string FormatStatus(IngestionJobStatus status) => status switch
|
||||
{
|
||||
IngestionJobStatus.Queued => "queued",
|
||||
IngestionJobStatus.Running => "running",
|
||||
IngestionJobStatus.Completed => "completed",
|
||||
IngestionJobStatus.Failed => "failed",
|
||||
IngestionJobStatus.Cancelled => "cancelled",
|
||||
_ => "paused"
|
||||
};
|
||||
|
||||
private static IngestionJobStatus ParseStatus(string status) => status.ToLowerInvariant() switch
|
||||
{
|
||||
"queued" => IngestionJobStatus.Queued,
|
||||
"running" => IngestionJobStatus.Running,
|
||||
"completed" => IngestionJobStatus.Completed,
|
||||
"failed" => IngestionJobStatus.Failed,
|
||||
"cancelled" => IngestionJobStatus.Cancelled,
|
||||
_ => IngestionJobStatus.Paused
|
||||
};
|
||||
|
||||
private static string GetSchemaName() => VexHubDataSource.DefaultSchemaName;
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using StellaOps.VexHub.Core;
|
||||
using StellaOps.VexHub.Core.Models;
|
||||
using StellaOps.VexLens.Models;
|
||||
|
||||
namespace StellaOps.VexHub.Persistence.Postgres.Repositories;
|
||||
|
||||
using EfModels = StellaOps.VexHub.Persistence.EfCore.Models;
|
||||
|
||||
/// <summary>
|
||||
/// PostgreSQL (EF Core) implementation of the VEX source repository.
|
||||
/// </summary>
|
||||
public sealed class PostgresVexSourceRepository : IVexSourceRepository
|
||||
{
|
||||
private const int CommandTimeoutSeconds = 30;
|
||||
|
||||
private readonly VexHubDataSource _dataSource;
|
||||
|
||||
public PostgresVexSourceRepository(VexHubDataSource dataSource)
|
||||
{
|
||||
_dataSource = dataSource;
|
||||
}
|
||||
|
||||
public async Task<VexSource> AddAsync(VexSource source, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = ToEntity(source);
|
||||
dbContext.Sources.Add(entity);
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
|
||||
return ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<VexSource> UpdateAsync(VexSource source, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.Sources.FirstOrDefaultAsync(s => s.SourceId == source.SourceId, cancellationToken);
|
||||
if (entity is null)
|
||||
{
|
||||
entity = ToEntity(source);
|
||||
dbContext.Sources.Add(entity);
|
||||
}
|
||||
else
|
||||
{
|
||||
Apply(source, entity);
|
||||
}
|
||||
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
return ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<VexSource?> GetByIdAsync(string sourceId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.Sources
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(s => s.SourceId == sourceId, cancellationToken);
|
||||
|
||||
return entity is null ? null : ToModel(entity);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<VexSource>> GetAllAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entities = await dbContext.Sources
|
||||
.AsNoTracking()
|
||||
.OrderBy(s => s.Name)
|
||||
.ThenBy(s => s.SourceId)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return entities.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<VexSource>> GetDueForPollingAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
var entities = await dbContext.Sources
|
||||
.AsNoTracking()
|
||||
.Where(s => s.IsEnabled && s.PollingIntervalSeconds.HasValue)
|
||||
.OrderBy(s => s.LastPolledAt)
|
||||
.ThenBy(s => s.SourceId)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
return entities
|
||||
.Where(entity => entity.LastPolledAt is null
|
||||
|| entity.LastPolledAt.Value.AddSeconds(entity.PollingIntervalSeconds ?? 0) <= nowUtc)
|
||||
.Select(ToModel)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public async Task UpdateLastPolledAsync(
|
||||
string sourceId,
|
||||
DateTimeOffset timestamp,
|
||||
string? errorMessage = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var entity = await dbContext.Sources.FirstOrDefaultAsync(s => s.SourceId == sourceId, cancellationToken);
|
||||
if (entity is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
entity.LastPolledAt = timestamp.UtcDateTime;
|
||||
entity.LastErrorMessage = errorMessage;
|
||||
entity.UpdatedAt = timestamp.UtcDateTime;
|
||||
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteAsync(string sourceId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var connection = await _dataSource.OpenSystemConnectionAsync(cancellationToken);
|
||||
await using var dbContext = VexHubDbContextFactory.Create(connection, CommandTimeoutSeconds, GetSchemaName());
|
||||
|
||||
var deleted = await dbContext.Sources
|
||||
.Where(s => s.SourceId == sourceId)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
|
||||
return deleted > 0;
|
||||
}
|
||||
|
||||
private static EfModels.VexSource ToEntity(VexSource model) => new()
|
||||
{
|
||||
SourceId = model.SourceId,
|
||||
Name = model.Name,
|
||||
SourceUri = model.SourceUri,
|
||||
SourceFormat = FormatSourceFormat(model.SourceFormat),
|
||||
IssuerCategory = model.IssuerCategory.HasValue ? FormatIssuerCategory(model.IssuerCategory.Value) : null,
|
||||
TrustTier = FormatTrustTier(model.TrustTier),
|
||||
IsEnabled = model.IsEnabled,
|
||||
PollingIntervalSeconds = model.PollingIntervalSeconds,
|
||||
LastPolledAt = model.LastPolledAt?.UtcDateTime,
|
||||
LastErrorMessage = model.LastErrorMessage,
|
||||
Config = "{}",
|
||||
CreatedAt = model.CreatedAt.UtcDateTime,
|
||||
UpdatedAt = model.UpdatedAt?.UtcDateTime
|
||||
};
|
||||
|
||||
private static void Apply(VexSource model, EfModels.VexSource entity)
|
||||
{
|
||||
entity.Name = model.Name;
|
||||
entity.SourceUri = model.SourceUri;
|
||||
entity.SourceFormat = FormatSourceFormat(model.SourceFormat);
|
||||
entity.IssuerCategory = model.IssuerCategory.HasValue ? FormatIssuerCategory(model.IssuerCategory.Value) : null;
|
||||
entity.TrustTier = FormatTrustTier(model.TrustTier);
|
||||
entity.IsEnabled = model.IsEnabled;
|
||||
entity.PollingIntervalSeconds = model.PollingIntervalSeconds;
|
||||
entity.LastPolledAt = model.LastPolledAt?.UtcDateTime;
|
||||
entity.LastErrorMessage = model.LastErrorMessage;
|
||||
entity.UpdatedAt = model.UpdatedAt?.UtcDateTime ?? DateTime.UtcNow;
|
||||
entity.Config ??= "{}";
|
||||
}
|
||||
|
||||
private static VexSource ToModel(EfModels.VexSource entity) => new()
|
||||
{
|
||||
SourceId = entity.SourceId,
|
||||
Name = entity.Name,
|
||||
SourceUri = entity.SourceUri,
|
||||
SourceFormat = ParseSourceFormat(entity.SourceFormat),
|
||||
IssuerCategory = string.IsNullOrWhiteSpace(entity.IssuerCategory) ? null : ParseIssuerCategory(entity.IssuerCategory),
|
||||
TrustTier = ParseTrustTier(entity.TrustTier),
|
||||
IsEnabled = entity.IsEnabled,
|
||||
PollingIntervalSeconds = entity.PollingIntervalSeconds,
|
||||
LastPolledAt = entity.LastPolledAt.HasValue ? new DateTimeOffset(entity.LastPolledAt.Value, TimeSpan.Zero) : null,
|
||||
LastErrorMessage = entity.LastErrorMessage,
|
||||
CreatedAt = new DateTimeOffset(entity.CreatedAt, TimeSpan.Zero),
|
||||
UpdatedAt = entity.UpdatedAt.HasValue ? new DateTimeOffset(entity.UpdatedAt.Value, TimeSpan.Zero) : null
|
||||
};
|
||||
|
||||
private static string FormatSourceFormat(VexSourceFormat sourceFormat) => sourceFormat switch
|
||||
{
|
||||
VexSourceFormat.OpenVex => "OPENVEX",
|
||||
VexSourceFormat.CsafVex => "CSAF_VEX",
|
||||
VexSourceFormat.CycloneDxVex => "CYCLONEDX_VEX",
|
||||
VexSourceFormat.SpdxVex => "SPDX_VEX",
|
||||
VexSourceFormat.StellaOps => "STELLAOPS",
|
||||
_ => "UNKNOWN"
|
||||
};
|
||||
|
||||
private static VexSourceFormat ParseSourceFormat(string sourceFormat) => sourceFormat.ToUpperInvariant() switch
|
||||
{
|
||||
"OPENVEX" => VexSourceFormat.OpenVex,
|
||||
"CSAF_VEX" => VexSourceFormat.CsafVex,
|
||||
"CYCLONEDX_VEX" => VexSourceFormat.CycloneDxVex,
|
||||
"SPDX_VEX" => VexSourceFormat.SpdxVex,
|
||||
"STELLAOPS" => VexSourceFormat.StellaOps,
|
||||
_ => VexSourceFormat.Unknown
|
||||
};
|
||||
|
||||
private static string FormatIssuerCategory(IssuerCategory issuerCategory) => issuerCategory switch
|
||||
{
|
||||
IssuerCategory.Vendor => "VENDOR",
|
||||
IssuerCategory.Distributor => "DISTRIBUTOR",
|
||||
IssuerCategory.Community => "COMMUNITY",
|
||||
IssuerCategory.Internal => "INTERNAL",
|
||||
IssuerCategory.Aggregator => "AGGREGATOR",
|
||||
_ => issuerCategory.ToString().ToUpperInvariant()
|
||||
};
|
||||
|
||||
private static IssuerCategory ParseIssuerCategory(string issuerCategory) => issuerCategory.ToUpperInvariant() switch
|
||||
{
|
||||
"VENDOR" => IssuerCategory.Vendor,
|
||||
"DISTRIBUTOR" => IssuerCategory.Distributor,
|
||||
"COMMUNITY" => IssuerCategory.Community,
|
||||
"INTERNAL" => IssuerCategory.Internal,
|
||||
"AGGREGATOR" => IssuerCategory.Aggregator,
|
||||
_ => throw new ArgumentException($"Unknown issuer category: {issuerCategory}")
|
||||
};
|
||||
|
||||
private static string FormatTrustTier(TrustTier trustTier) => trustTier switch
|
||||
{
|
||||
TrustTier.Authoritative => "AUTHORITATIVE",
|
||||
TrustTier.Trusted => "TRUSTED",
|
||||
TrustTier.Untrusted => "UNTRUSTED",
|
||||
_ => "UNKNOWN"
|
||||
};
|
||||
|
||||
private static TrustTier ParseTrustTier(string trustTier) => trustTier.ToUpperInvariant() switch
|
||||
{
|
||||
"AUTHORITATIVE" => TrustTier.Authoritative,
|
||||
"TRUSTED" => TrustTier.Trusted,
|
||||
"UNTRUSTED" => TrustTier.Untrusted,
|
||||
_ => TrustTier.Unknown
|
||||
};
|
||||
|
||||
private static string GetSchemaName() => VexHubDataSource.DefaultSchemaName;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Npgsql;
|
||||
using StellaOps.VexHub.Persistence.EfCore.CompiledModels;
|
||||
@@ -22,7 +23,8 @@ internal static class VexHubDbContextFactory
|
||||
var optionsBuilder = new DbContextOptionsBuilder<VexHubDbContext>()
|
||||
.UseNpgsql(connection, npgsql => npgsql.CommandTimeout(commandTimeoutSeconds));
|
||||
|
||||
if (string.Equals(normalizedSchema, VexHubDataSource.DefaultSchemaName, StringComparison.Ordinal))
|
||||
if (string.Equals(normalizedSchema, VexHubDataSource.DefaultSchemaName, StringComparison.Ordinal)
|
||||
&& VexHubDbContextModel.Instance.GetEntityTypes().Any())
|
||||
{
|
||||
// Use the static compiled model when schema mapping matches the default model.
|
||||
optionsBuilder.UseModel(VexHubDbContextModel.Instance);
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using StellaOps.VexHub.Persistence.EfCore.Context;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.VexHub.WebService.Tests.Integration;
|
||||
|
||||
public sealed class VexEfModelCompatibilityTests
|
||||
{
|
||||
[Fact]
|
||||
public void VexHubDbContext_builds_runtime_model_without_tsvector_mapping_errors()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<VexHubDbContext>()
|
||||
.UseNpgsql("Host=localhost;Database=vexhub_test;Username=stella;Password=stella")
|
||||
.Options;
|
||||
|
||||
using var dbContext = new VexHubDbContext(options);
|
||||
|
||||
var act = () => _ = dbContext.Model;
|
||||
|
||||
act.Should().NotThrow();
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,11 @@ using System.Text.Json;
|
||||
using System.Collections.Concurrent;
|
||||
using FluentAssertions;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.VexHub.Core;
|
||||
using StellaOps.VexHub.Core.Models;
|
||||
using StellaOps.VexLens.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.VexHub.WebService.Tests.Integration;
|
||||
@@ -16,12 +18,25 @@ namespace StellaOps.VexHub.WebService.Tests.Integration;
|
||||
/// </summary>
|
||||
public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFactory<StellaOps.VexHub.WebService.Program>>
|
||||
{
|
||||
private const string TestApiKey = "integration-test-key";
|
||||
private static readonly DateTimeOffset FixedNow = DateTimeOffset.Parse("2026-03-13T12:00:00Z");
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public VexExportCompatibilityTests(WebApplicationFactory<StellaOps.VexHub.WebService.Program> factory)
|
||||
{
|
||||
_client = factory.WithWebHostBuilder(builder =>
|
||||
{
|
||||
builder.ConfigureAppConfiguration((_, config) =>
|
||||
{
|
||||
config.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
[$"VexHub:ApiKeys:{TestApiKey}:KeyId"] = "integration-test",
|
||||
[$"VexHub:ApiKeys:{TestApiKey}:ClientId"] = "integration-suite",
|
||||
[$"VexHub:ApiKeys:{TestApiKey}:ClientName"] = "Integration Suite",
|
||||
[$"VexHub:ApiKeys:{TestApiKey}:Scopes:0"] = "VexHub.Read",
|
||||
[$"VexHub:ApiKeys:{TestApiKey}:Scopes:1"] = "VexHub.Admin",
|
||||
});
|
||||
});
|
||||
builder.ConfigureServices(services =>
|
||||
{
|
||||
services.AddSingleton<IVexSourceRepository, InMemoryVexSourceRepository>();
|
||||
@@ -30,6 +45,8 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
services.AddSingleton<IVexStatementRepository, InMemoryVexStatementRepository>();
|
||||
});
|
||||
}).CreateClient();
|
||||
_client.DefaultRequestHeaders.Add("X-Api-Key", TestApiKey);
|
||||
_client.DefaultRequestHeaders.Add("X-StellaOps-Tenant", "test-tenant");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -118,6 +135,21 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
response.StatusCode.Should().BeOneOf(HttpStatusCode.OK, HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StatsEndpoint_ReturnsDashboardCompatibleShape()
|
||||
{
|
||||
var response = await _client.GetAsync("/api/v1/vex/stats");
|
||||
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
|
||||
var payload = JsonDocument.Parse(await response.Content.ReadAsStringAsync()).RootElement;
|
||||
payload.GetProperty("totalStatements").GetInt64().Should().BeGreaterThan(0);
|
||||
payload.GetProperty("byStatus").GetProperty("affected").GetInt64().Should().BeGreaterThan(0);
|
||||
payload.GetProperty("bySource").GetProperty("vendor").GetInt64().Should().BeGreaterThan(0);
|
||||
payload.GetProperty("recentActivity").GetArrayLength().Should().BeGreaterThan(0);
|
||||
payload.GetProperty("trends").GetArrayLength().Should().Be(7);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SourceEndpoint_ReturnsValidResponse()
|
||||
{
|
||||
@@ -207,6 +239,45 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, VexSource> _sources = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public InMemoryVexSourceRepository()
|
||||
{
|
||||
Seed(new VexSource
|
||||
{
|
||||
SourceId = "redhat-csaf",
|
||||
Name = "Red Hat CSAF",
|
||||
SourceFormat = VexSourceFormat.CsafVex,
|
||||
IssuerCategory = IssuerCategory.Vendor,
|
||||
TrustTier = TrustTier.Authoritative,
|
||||
CreatedAt = FixedNow.AddDays(-30),
|
||||
UpdatedAt = FixedNow.AddDays(-1),
|
||||
IsEnabled = true,
|
||||
});
|
||||
Seed(new VexSource
|
||||
{
|
||||
SourceId = "internal-vex",
|
||||
Name = "Internal VEX",
|
||||
SourceFormat = VexSourceFormat.OpenVex,
|
||||
IssuerCategory = IssuerCategory.Internal,
|
||||
TrustTier = TrustTier.Trusted,
|
||||
CreatedAt = FixedNow.AddDays(-14),
|
||||
UpdatedAt = FixedNow.AddHours(-6),
|
||||
IsEnabled = true,
|
||||
});
|
||||
Seed(new VexSource
|
||||
{
|
||||
SourceId = "osv-community",
|
||||
Name = "OSV Community",
|
||||
SourceFormat = VexSourceFormat.OpenVex,
|
||||
IssuerCategory = IssuerCategory.Community,
|
||||
TrustTier = TrustTier.Trusted,
|
||||
CreatedAt = FixedNow.AddDays(-21),
|
||||
UpdatedAt = FixedNow.AddHours(-12),
|
||||
IsEnabled = true,
|
||||
});
|
||||
}
|
||||
|
||||
private void Seed(VexSource source) => _sources[source.SourceId] = source;
|
||||
|
||||
public Task<VexSource> AddAsync(VexSource source, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_sources[source.SourceId] = source;
|
||||
@@ -248,7 +319,7 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
{
|
||||
LastPolledAt = timestamp,
|
||||
LastErrorMessage = errorMessage,
|
||||
UpdatedAt = DateTimeOffset.UtcNow
|
||||
UpdatedAt = timestamp
|
||||
};
|
||||
}
|
||||
|
||||
@@ -331,7 +402,7 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
ResolutionStatus = status,
|
||||
ResolutionMethod = resolutionMethod,
|
||||
WinningStatementId = winningStatementId,
|
||||
ResolvedAt = DateTimeOffset.UtcNow
|
||||
ResolvedAt = FixedNow
|
||||
};
|
||||
}
|
||||
|
||||
@@ -446,7 +517,7 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
StatementsIngested = statementsIngested,
|
||||
StatementsDeduplicated = statementsDeduplicated,
|
||||
ConflictsDetected = conflictsDetected,
|
||||
CompletedAt = DateTimeOffset.UtcNow
|
||||
CompletedAt = FixedNow
|
||||
};
|
||||
}
|
||||
|
||||
@@ -461,7 +532,7 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
{
|
||||
Status = IngestionJobStatus.Failed,
|
||||
ErrorMessage = errorMessage,
|
||||
CompletedAt = DateTimeOffset.UtcNow
|
||||
CompletedAt = FixedNow
|
||||
};
|
||||
}
|
||||
|
||||
@@ -473,6 +544,57 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
{
|
||||
private readonly ConcurrentDictionary<Guid, AggregatedVexStatement> _statements = new();
|
||||
|
||||
public InMemoryVexStatementRepository()
|
||||
{
|
||||
Seed(new AggregatedVexStatement
|
||||
{
|
||||
Id = Guid.Parse("11111111-1111-1111-1111-111111111111"),
|
||||
SourceStatementId = "stmt-redhat-001",
|
||||
SourceId = "redhat-csaf",
|
||||
SourceDocumentId = "doc-redhat-001",
|
||||
VulnerabilityId = "CVE-2026-1001",
|
||||
ProductKey = "pkg:oci/acme/api@sha256:1111",
|
||||
Status = VexStatus.Affected,
|
||||
VerificationStatus = VerificationStatus.Verified,
|
||||
IsFlagged = false,
|
||||
IngestedAt = FixedNow.AddDays(-1),
|
||||
SourceUpdatedAt = FixedNow.AddHours(-10),
|
||||
ContentDigest = "sha256:redhat-001",
|
||||
});
|
||||
Seed(new AggregatedVexStatement
|
||||
{
|
||||
Id = Guid.Parse("22222222-2222-2222-2222-222222222222"),
|
||||
SourceStatementId = "stmt-internal-001",
|
||||
SourceId = "internal-vex",
|
||||
SourceDocumentId = "doc-internal-001",
|
||||
VulnerabilityId = "CVE-2026-1002",
|
||||
ProductKey = "pkg:oci/acme/web@sha256:2222",
|
||||
Status = VexStatus.NotAffected,
|
||||
VerificationStatus = VerificationStatus.Pending,
|
||||
IsFlagged = false,
|
||||
IngestedAt = FixedNow.AddDays(-2),
|
||||
SourceUpdatedAt = FixedNow.AddHours(-20),
|
||||
ContentDigest = "sha256:internal-001",
|
||||
});
|
||||
Seed(new AggregatedVexStatement
|
||||
{
|
||||
Id = Guid.Parse("33333333-3333-3333-3333-333333333333"),
|
||||
SourceStatementId = "stmt-community-001",
|
||||
SourceId = "osv-community",
|
||||
SourceDocumentId = "doc-community-001",
|
||||
VulnerabilityId = "CVE-2026-1003",
|
||||
ProductKey = "pkg:oci/acme/worker@sha256:3333",
|
||||
Status = VexStatus.Fixed,
|
||||
VerificationStatus = VerificationStatus.None,
|
||||
IsFlagged = true,
|
||||
IngestedAt = FixedNow.AddDays(-3),
|
||||
SourceUpdatedAt = FixedNow.AddDays(-1).AddHours(-2),
|
||||
ContentDigest = "sha256:community-001",
|
||||
});
|
||||
}
|
||||
|
||||
private void Seed(AggregatedVexStatement statement) => _statements[statement.Id] = statement;
|
||||
|
||||
public Task<AggregatedVexStatement> UpsertAsync(AggregatedVexStatement statement, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_statements[statement.Id] = statement;
|
||||
@@ -502,7 +624,8 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
int? offset = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(Array.Empty<AggregatedVexStatement>());
|
||||
var results = ApplyFilter(new VexStatementFilter { VulnerabilityId = cveId }, limit, offset);
|
||||
return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(results);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AggregatedVexStatement>> GetByPackageAsync(
|
||||
@@ -511,7 +634,8 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
int? offset = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(Array.Empty<AggregatedVexStatement>());
|
||||
var results = ApplyFilter(new VexStatementFilter { ProductKey = purl }, limit, offset);
|
||||
return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(results);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AggregatedVexStatement>> GetBySourceAsync(
|
||||
@@ -520,14 +644,15 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
int? offset = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(Array.Empty<AggregatedVexStatement>());
|
||||
var results = ApplyFilter(new VexStatementFilter { SourceId = sourceId }, limit, offset);
|
||||
return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(results);
|
||||
}
|
||||
|
||||
public Task<bool> ExistsByDigestAsync(string contentDigest, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(false);
|
||||
=> Task.FromResult(_statements.Values.Any(statement => string.Equals(statement.ContentDigest, contentDigest, StringComparison.OrdinalIgnoreCase)));
|
||||
|
||||
public Task<long> GetCountAsync(VexStatementFilter? filter = null, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0L);
|
||||
=> Task.FromResult((long)ApplyFilter(filter, null, null).Count);
|
||||
|
||||
public Task<IReadOnlyList<AggregatedVexStatement>> SearchAsync(
|
||||
VexStatementFilter filter,
|
||||
@@ -535,13 +660,85 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
|
||||
int? offset = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(Array.Empty<AggregatedVexStatement>());
|
||||
var results = ApplyFilter(filter, limit, offset);
|
||||
return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(results);
|
||||
}
|
||||
|
||||
public Task FlagStatementAsync(Guid id, string reason, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
{
|
||||
if (_statements.TryGetValue(id, out var statement))
|
||||
{
|
||||
_statements[id] = statement with { IsFlagged = true, FlagReason = reason };
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<int> DeleteBySourceAsync(string sourceId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(0);
|
||||
{
|
||||
var removed = _statements.Values
|
||||
.Where(statement => string.Equals(statement.SourceId, sourceId, StringComparison.OrdinalIgnoreCase))
|
||||
.Select(statement => statement.Id)
|
||||
.ToArray();
|
||||
|
||||
foreach (var id in removed)
|
||||
{
|
||||
_statements.TryRemove(id, out _);
|
||||
}
|
||||
|
||||
return Task.FromResult(removed.Length);
|
||||
}
|
||||
|
||||
private List<AggregatedVexStatement> ApplyFilter(VexStatementFilter? filter, int? limit, int? offset)
|
||||
{
|
||||
IEnumerable<AggregatedVexStatement> query = _statements.Values
|
||||
.OrderByDescending(statement => statement.IngestedAt)
|
||||
.ThenBy(statement => statement.SourceId, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (filter is not null)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(filter.SourceId))
|
||||
{
|
||||
query = query.Where(statement => string.Equals(statement.SourceId, filter.SourceId, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(filter.VulnerabilityId))
|
||||
{
|
||||
query = query.Where(statement => string.Equals(statement.VulnerabilityId, filter.VulnerabilityId, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(filter.ProductKey))
|
||||
{
|
||||
query = query.Where(statement => string.Equals(statement.ProductKey, filter.ProductKey, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
if (filter.Status.HasValue)
|
||||
{
|
||||
query = query.Where(statement => statement.Status == filter.Status.Value);
|
||||
}
|
||||
|
||||
if (filter.VerificationStatus.HasValue)
|
||||
{
|
||||
query = query.Where(statement => statement.VerificationStatus == filter.VerificationStatus.Value);
|
||||
}
|
||||
|
||||
if (filter.IsFlagged.HasValue)
|
||||
{
|
||||
query = query.Where(statement => statement.IsFlagged == filter.IsFlagged.Value);
|
||||
}
|
||||
}
|
||||
|
||||
if (offset.HasValue)
|
||||
{
|
||||
query = query.Skip(offset.Value);
|
||||
}
|
||||
|
||||
if (limit.HasValue)
|
||||
{
|
||||
query = query.Take(limit.Value);
|
||||
}
|
||||
|
||||
return query.ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using StellaOps.VexHub.Core;
|
||||
using StellaOps.VexHub.Persistence.Extensions;
|
||||
using StellaOps.VexHub.Persistence.Postgres.Repositories;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.VexHub.WebService.Tests.Integration;
|
||||
|
||||
public sealed class VexPersistenceRegistrationTests
|
||||
{
|
||||
[Fact]
|
||||
public void AddVexHubPersistence_registers_all_core_repository_contracts()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection()
|
||||
.Build();
|
||||
|
||||
services.AddVexHubPersistence(configuration);
|
||||
|
||||
services.Should().ContainSingle(descriptor =>
|
||||
descriptor.ServiceType == typeof(IVexSourceRepository)
|
||||
&& descriptor.ImplementationType == typeof(PostgresVexSourceRepository)
|
||||
&& descriptor.Lifetime == ServiceLifetime.Scoped);
|
||||
services.Should().ContainSingle(descriptor =>
|
||||
descriptor.ServiceType == typeof(IVexConflictRepository)
|
||||
&& descriptor.ImplementationType == typeof(PostgresVexConflictRepository)
|
||||
&& descriptor.Lifetime == ServiceLifetime.Scoped);
|
||||
services.Should().ContainSingle(descriptor =>
|
||||
descriptor.ServiceType == typeof(IVexIngestionJobRepository)
|
||||
&& descriptor.ImplementationType == typeof(PostgresVexIngestionJobRepository)
|
||||
&& descriptor.Lifetime == ServiceLifetime.Scoped);
|
||||
services.Should().ContainSingle(descriptor =>
|
||||
descriptor.ServiceType == typeof(IVexStatementRepository)
|
||||
&& descriptor.ImplementationType == typeof(PostgresVexStatementRepository)
|
||||
&& descriptor.Lifetime == ServiceLifetime.Scoped);
|
||||
services.Should().ContainSingle(descriptor =>
|
||||
descriptor.ServiceType == typeof(IVexProvenanceRepository)
|
||||
&& descriptor.ImplementationType == typeof(PostgresVexProvenanceRepository)
|
||||
&& descriptor.Lifetime == ServiceLifetime.Scoped);
|
||||
services.Should().Contain(descriptor =>
|
||||
descriptor.ServiceType == typeof(IHostedService));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user