stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

Repair live canonical migrations and scanner cache bootstrap

Browse Source
This commit is contained in:
master
2026-03-09 21:56:41 +02:00
parent 00bf2fa99a
commit dfd22281ed
21 changed files with 1018 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Extensions/ConcelierPersistenceExtensions.cs
View File

@@ -11,6 +11,7 @@ using StellaOps.Concelier.Merge.Backport;
using StellaOps.Concelier.Persistence.Postgres;
using StellaOps.Concelier.Persistence.Postgres.Advisories;
using StellaOps.Concelier.Persistence.Postgres.Repositories;
using StellaOps.Infrastructure.Postgres.Migrations;
using StellaOps.Infrastructure.Postgres.Options;
using StorageContracts = StellaOps.Concelier.Storage;
@@ -35,6 +36,10 @@ public static class ConcelierPersistenceExtensions
{
services.Configure<PostgresOptions>(sectionName, configuration.GetSection(sectionName));
services.AddSingleton<ConcelierDataSource>();
services.AddStartupMigrations(
ConcelierDataSource.DefaultSchemaName,
"Concelier.Storage",
typeof(ConcelierDataSource).Assembly);
// Register repositories
services.AddScoped<IAdvisoryRepository, AdvisoryRepository>();
@@ -83,6 +88,10 @@ public static class ConcelierPersistenceExtensions
{
services.Configure(configureOptions);
services.AddSingleton<ConcelierDataSource>();
services.AddStartupMigrations(
ConcelierDataSource.DefaultSchemaName,
"Concelier.Storage",
typeof(ConcelierDataSource).Assembly);
// Register repositories
services.AddScoped<IAdvisoryRepository, AdvisoryRepository>();

365
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/PostgresCanonicalAdvisoryStore.cs Normal file
View File

@@ -0,0 +1,365 @@
using System.Text.Json;
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Concelier.Merge.Backport;
using StellaOps.Concelier.Persistence.Postgres.Models;
using StellaOps.Concelier.Persistence.Postgres.Repositories;
using MergeHashInput = StellaOps.Concelier.Merge.Identity.MergeHashInput;
namespace StellaOps.Concelier.Persistence.Postgres;
public sealed class MergeHashCalculatorAdapter : IMergeHashCalculator
{
private readonly StellaOps.Concelier.Merge.Identity.IMergeHashCalculator inner;
public MergeHashCalculatorAdapter(StellaOps.Concelier.Merge.Identity.IMergeHashCalculator inner)
{
this.inner = inner ?? throw new ArgumentNullException(nameof(inner));
}
public string ComputeMergeHash(StellaOps.Concelier.Core.Canonical.MergeHashInput input)
{
ArgumentNullException.ThrowIfNull(input);
return inner.ComputeMergeHash(new MergeHashInput
{
Cve = input.Cve,
AffectsKey = input.AffectsKey,
VersionRange = input.VersionRange,
Weaknesses = input.Weaknesses,
PatchLineage = input.PatchLineage
});
}
}
public sealed class PostgresCanonicalAdvisoryStore : ICanonicalAdvisoryStore
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly IAdvisoryCanonicalRepository advisoryRepository;
private readonly ISourceRepository sourceRepository;
private readonly IProvenanceScopeStore provenanceScopeStore;
public PostgresCanonicalAdvisoryStore(
IAdvisoryCanonicalRepository advisoryRepository,
ISourceRepository sourceRepository,
IProvenanceScopeStore provenanceScopeStore)
{
this.advisoryRepository = advisoryRepository ?? throw new ArgumentNullException(nameof(advisoryRepository));
this.sourceRepository = sourceRepository ?? throw new ArgumentNullException(nameof(sourceRepository));
this.provenanceScopeStore = provenanceScopeStore ?? throw new ArgumentNullException(nameof(provenanceScopeStore));
}
public async Task<CanonicalAdvisory?> GetByIdAsync(Guid id, CancellationToken ct = default)
{
var entity = await advisoryRepository.GetByIdAsync(id, ct).ConfigureAwait(false);
return entity is null ? null : await MapCanonicalAsync(entity, ct).ConfigureAwait(false);
}
public async Task<CanonicalAdvisory?> GetByMergeHashAsync(string mergeHash, CancellationToken ct = default)
{
var entity = await advisoryRepository.GetByMergeHashAsync(mergeHash, ct).ConfigureAwait(false);
return entity is null ? null : await MapCanonicalAsync(entity, ct).ConfigureAwait(false);
}
public async Task<IReadOnlyList<CanonicalAdvisory>> GetByCveAsync(string cve, CancellationToken ct = default)
{
var entities = await advisoryRepository.GetByCveAsync(cve, ct).ConfigureAwait(false);
return await MapCanonicalsAsync(entities, ct).ConfigureAwait(false);
}
public async Task<IReadOnlyList<CanonicalAdvisory>> GetByArtifactAsync(string artifactKey, CancellationToken ct = default)
{
var entities = await advisoryRepository.GetByAffectsKeyAsync(artifactKey, ct).ConfigureAwait(false);
return await MapCanonicalsAsync(entities, ct).ConfigureAwait(false);
}
public async Task<PagedResult<CanonicalAdvisory>> QueryAsync(CanonicalQueryOptions options, CancellationToken ct = default)
{
var result = await advisoryRepository.QueryAsync(options, ct).ConfigureAwait(false);
var items = await MapCanonicalsAsync(result.Items, ct).ConfigureAwait(false);
return new PagedResult<CanonicalAdvisory>
{
Items = items,
TotalCount = result.TotalCount,
Offset = result.Offset,
Limit = result.Limit
};
}
public async Task<Guid> UpsertCanonicalAsync(UpsertCanonicalRequest request, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(request);
var entity = new AdvisoryCanonicalEntity
{
Id = Guid.Empty,
Cve = request.Cve,
AffectsKey = request.AffectsKey,
VersionRange = request.VersionRangeJson,
Weakness = request.Weaknesses.ToArray(),
MergeHash = request.MergeHash,
Status = CanonicalStatus.Active.ToString().ToLowerInvariant(),
Severity = request.Severity,
EpssScore = request.EpssScore,
ExploitKnown = request.ExploitKnown,
Title = request.Title,
Summary = request.Summary
};
return await advisoryRepository.UpsertAsync(entity, ct).ConfigureAwait(false);
}
public Task UpdateStatusAsync(Guid id, CanonicalStatus status, CancellationToken ct = default)
=> advisoryRepository.UpdateStatusAsync(id, status.ToString().ToLowerInvariant(), ct);
public Task<long> CountAsync(CancellationToken ct = default)
=> advisoryRepository.CountAsync(ct);
public async Task<SourceEdgeResult> AddSourceEdgeAsync(AddSourceEdgeRequest request, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(request);
var existed = await SourceEdgeExistsAsync(request.CanonicalId, request.SourceId, request.SourceDocHash, ct)
.ConfigureAwait(false);
var edgeId = await advisoryRepository.AddSourceEdgeAsync(
new AdvisorySourceEdgeEntity
{
Id = Guid.Empty,
CanonicalId = request.CanonicalId,
SourceId = request.SourceId,
SourceAdvisoryId = request.SourceAdvisoryId,
SourceDocHash = request.SourceDocHash,
VendorStatus = request.VendorStatus?.ToString().ToLowerInvariant(),
PrecedenceRank = request.PrecedenceRank,
DsseEnvelope = request.DsseEnvelopeJson,
RawPayload = request.RawPayloadJson,
FetchedAt = request.FetchedAt
},
ct)
.ConfigureAwait(false);
return existed ? SourceEdgeResult.Existing(edgeId) : SourceEdgeResult.Created(edgeId);
}
public async Task<IReadOnlyList<SourceEdge>> GetSourceEdgesAsync(Guid canonicalId, CancellationToken ct = default)
{
var edges = await advisoryRepository.GetSourceEdgesAsync(canonicalId, ct).ConfigureAwait(false);
return await MapSourceEdgesAsync(edges, ct).ConfigureAwait(false);
}
public async Task<bool> SourceEdgeExistsAsync(Guid canonicalId, Guid sourceId, string docHash, CancellationToken ct = default)
{
var edges = await advisoryRepository.GetSourceEdgesAsync(canonicalId, ct).ConfigureAwait(false);
return edges.Any(edge =>
edge.SourceId == sourceId &&
string.Equals(edge.SourceDocHash, docHash, StringComparison.Ordinal));
}
public async Task<IReadOnlyList<ProvenanceScopeDto>> GetProvenanceScopesAsync(Guid canonicalId, CancellationToken ct = default)
{
var scopes = await provenanceScopeStore.GetByCanonicalIdAsync(canonicalId, ct).ConfigureAwait(false);
return scopes.Select(MapProvenanceScope).ToList();
}
public async Task<Guid> ResolveSourceIdAsync(string sourceKey, CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey);
var existing = await sourceRepository.GetByKeyAsync(sourceKey, ct).ConfigureAwait(false);
if (existing is not null)
{
return existing.Id;
}
var created = await sourceRepository.UpsertAsync(
new SourceEntity
{
Id = Guid.NewGuid(),
Key = sourceKey.Trim(),
Name = sourceKey.Trim(),
SourceType = sourceKey.Trim(),
Priority = 100,
Enabled = true,
Config = "{}",
Metadata = "{}"
},
ct)
.ConfigureAwait(false);
return created.Id;
}
public async Task<int> GetSourcePrecedenceAsync(string sourceKey, CancellationToken ct = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceKey);
var source = await sourceRepository.GetByKeyAsync(sourceKey, ct).ConfigureAwait(false);
return source?.Priority ?? 100;
}
private async Task<IReadOnlyList<CanonicalAdvisory>> MapCanonicalsAsync(
IReadOnlyList<AdvisoryCanonicalEntity> entities,
CancellationToken ct)
{
var results = new List<CanonicalAdvisory>(entities.Count);
foreach (var entity in entities)
{
results.Add(await MapCanonicalAsync(entity, ct).ConfigureAwait(false));
}
return results;
}
private async Task<CanonicalAdvisory> MapCanonicalAsync(AdvisoryCanonicalEntity entity, CancellationToken ct)
{
var sourceEdges = await advisoryRepository.GetSourceEdgesAsync(entity.Id, ct).ConfigureAwait(false);
var provenanceScopes = await provenanceScopeStore.GetByCanonicalIdAsync(entity.Id, ct).ConfigureAwait(false);
return new CanonicalAdvisory
{
Id = entity.Id,
Cve = entity.Cve,
AffectsKey = entity.AffectsKey,
VersionRange = ParseVersionRange(entity.VersionRange),
Weaknesses = entity.Weakness,
MergeHash = entity.MergeHash,
Status = ParseCanonicalStatus(entity.Status),
Severity = entity.Severity,
EpssScore = entity.EpssScore,
ExploitKnown = entity.ExploitKnown,
Title = entity.Title,
Summary = entity.Summary,
CreatedAt = entity.CreatedAt,
UpdatedAt = entity.UpdatedAt,
SourceEdges = await MapSourceEdgesAsync(sourceEdges, ct).ConfigureAwait(false),
ProvenanceScopes = provenanceScopes.Select(MapProvenanceScope).ToList()
};
}
private async Task<IReadOnlyList<SourceEdge>> MapSourceEdgesAsync(
IReadOnlyList<AdvisorySourceEdgeEntity> entities,
CancellationToken ct)
{
var results = new List<SourceEdge>(entities.Count);
foreach (var entity in entities)
{
var source = await sourceRepository.GetByIdAsync(entity.SourceId, ct).ConfigureAwait(false);
results.Add(new SourceEdge
{
Id = entity.Id,
CanonicalId = entity.CanonicalId,
SourceName = source?.Key ?? entity.SourceId.ToString("D"),
SourceAdvisoryId = entity.SourceAdvisoryId,
SourceDocHash = entity.SourceDocHash,
VendorStatus = ParseVendorStatus(entity.VendorStatus),
PrecedenceRank = entity.PrecedenceRank,
DsseEnvelope = ParseDsseEnvelope(entity.DsseEnvelope),
FetchedAt = entity.FetchedAt,
CreatedAt = entity.CreatedAt
});
}
return results;
}
private static CanonicalStatus ParseCanonicalStatus(string? status)
{
return status?.Trim().ToLowerInvariant() switch
{
"stub" => CanonicalStatus.Stub,
"withdrawn" => CanonicalStatus.Withdrawn,
_ => CanonicalStatus.Active
};
}
private static VendorStatus? ParseVendorStatus(string? status)
{
return status?.Trim().ToLowerInvariant() switch
{
"affected" => VendorStatus.Affected,
"not_affected" => VendorStatus.NotAffected,
"fixed" => VendorStatus.Fixed,
"under_investigation" => VendorStatus.UnderInvestigation,
_ => null
};
}
private static VersionRange? ParseVersionRange(string? versionRangeJson)
{
if (string.IsNullOrWhiteSpace(versionRangeJson))
{
return null;
}
var trimmed = versionRangeJson.Trim();
if (!trimmed.StartsWith("{", StringComparison.Ordinal))
{
return new VersionRange { RangeExpression = trimmed };
}
try
{
using var document = JsonDocument.Parse(trimmed);
var root = document.RootElement;
return new VersionRange
{
Introduced = GetProperty(root, "introduced"),
Fixed = GetProperty(root, "fixed"),
LastAffected = GetProperty(root, "lastAffected", "last_affected"),
RangeExpression = GetProperty(root, "rangeExpression", "range_expression")
};
}
catch (JsonException)
{
return new VersionRange { RangeExpression = trimmed };
}
}
private static DsseEnvelope? ParseDsseEnvelope(string? dsseEnvelopeJson)
{
if (string.IsNullOrWhiteSpace(dsseEnvelopeJson))
{
return null;
}
try
{
return JsonSerializer.Deserialize<DsseEnvelope>(dsseEnvelopeJson, JsonOptions);
}
catch (JsonException)
{
return null;
}
}
private static ProvenanceScopeDto MapProvenanceScope(ProvenanceScope scope)
{
return new ProvenanceScopeDto
{
Id = scope.Id,
DistroRelease = scope.DistroRelease,
BackportVersion = scope.BackportSemver,
PatchId = scope.PatchId,
PatchOrigin = scope.PatchOrigin?.ToString(),
EvidenceRef = scope.EvidenceRef,
Confidence = scope.Confidence,
UpdatedAt = scope.UpdatedAt
};
}
private static string? GetProperty(JsonElement element, params string[] names)
{
foreach (var name in names)
{
if (element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String)
{
return value.GetString();
}
}
return null;
}
}

106
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/AdvisoryCanonicalRepository.cs
View File

@@ -8,6 +8,7 @@
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Concelier.Persistence.Postgres.Models;
using StellaOps.Infrastructure.Postgres.Repositories;
using System.Runtime.CompilerServices;
@@ -131,6 +132,111 @@ public sealed class AdvisoryCanonicalRepository : RepositoryBase<ConcelierDataSo
ct);
}
public async Task<PagedResult<AdvisoryCanonicalEntity>> QueryAsync(
CanonicalQueryOptions options,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(options);
var filters = new List<string>();
if (!string.IsNullOrWhiteSpace(options.Cve))
{
filters.Add("cve = @cve");
}
if (!string.IsNullOrWhiteSpace(options.ArtifactKey))
{
filters.Add("affects_key = @affects_key");
}
if (!string.IsNullOrWhiteSpace(options.Severity))
{
filters.Add("severity = @severity");
}
if (options.Status is not null)
{
filters.Add("status = @status");
}
if (options.ExploitKnown is not null)
{
filters.Add("exploit_known = @exploit_known");
}
if (options.UpdatedSince is not null)
{
filters.Add("updated_at >= @updated_since");
}
var whereClause = filters.Count == 0
? string.Empty
: $"WHERE {string.Join(" AND ", filters)}";
var sql = $"""
SELECT id, cve, affects_key, version_range::text, weakness, merge_hash,
status, severity, epss_score, exploit_known, title, summary,
created_at, updated_at, COUNT(*) OVER() AS total_count
FROM vuln.advisory_canonical
{whereClause}
ORDER BY updated_at DESC, id ASC
OFFSET @offset
LIMIT @limit
""";
await using var connection = await DataSource.OpenSystemConnectionAsync(ct).ConfigureAwait(false);
await using var command = CreateCommand(sql, connection);
AddParameter(command, "offset", options.Offset);
AddParameter(command, "limit", options.Limit);
if (!string.IsNullOrWhiteSpace(options.Cve))
{
AddParameter(command, "cve", options.Cve);
}
if (!string.IsNullOrWhiteSpace(options.ArtifactKey))
{
AddParameter(command, "affects_key", options.ArtifactKey);
}
if (!string.IsNullOrWhiteSpace(options.Severity))
{
AddParameter(command, "severity", NormalizeSeverity(options.Severity));
}
if (options.Status is not null)
{
AddParameter(command, "status", options.Status.Value.ToString().ToLowerInvariant());
}
if (options.ExploitKnown is not null)
{
AddParameter(command, "exploit_known", options.ExploitKnown.Value);
}
if (options.UpdatedSince is not null)
{
AddParameter(command, "updated_since", options.UpdatedSince.Value);
}
var items = new List<AdvisoryCanonicalEntity>();
long totalCount = 0;
await using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
items.Add(MapCanonical(reader));
totalCount = reader.GetInt64(14);
}
return new PagedResult<AdvisoryCanonicalEntity>
{
Items = items,
TotalCount = totalCount,
Offset = options.Offset,
Limit = options.Limit
};
}
public async Task<Guid> UpsertAsync(AdvisoryCanonicalEntity entity, CancellationToken ct = default)
{
var normalizedSeverity = NormalizeSeverity(entity.Severity);

8
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/IAdvisoryCanonicalRepository.cs
View File

@@ -5,6 +5,7 @@
// Description: Repository interface for canonical advisory operations
// -----------------------------------------------------------------------------
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Concelier.Persistence.Postgres.Models;
namespace StellaOps.Concelier.Persistence.Postgres.Repositories;
@@ -44,6 +45,13 @@ public interface IAdvisoryCanonicalRepository
int limit = 1000,
CancellationToken ct = default);
/// <summary>
/// Queries canonical advisories with deterministic ordering and pagination.
/// </summary>
Task<PagedResult<AdvisoryCanonicalEntity>> QueryAsync(
CanonicalQueryOptions options,
CancellationToken ct = default);
/// <summary>
/// Upserts a canonical advisory (insert or update by merge_hash).
/// </summary>

18
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs
View File

@@ -6,12 +6,14 @@ using JpFlagsContracts = StellaOps.Concelier.Storage.JpFlags;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using PsirtContracts = StellaOps.Concelier.Storage.PsirtFlags;
using StellaOps.Concelier.Core.Canonical;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Merge.Backport;
using StellaOps.Concelier.Persistence.Postgres.Advisories;
using StellaOps.Concelier.Persistence.Postgres.Repositories;
using StellaOps.Concelier.SbomIntegration;
using StellaOps.Infrastructure.Postgres;
using StellaOps.Infrastructure.Postgres.Migrations;
using StellaOps.Infrastructure.Postgres.Options;
using StorageContracts = StellaOps.Concelier.Storage;
@@ -36,10 +38,18 @@ public static class ServiceCollectionExtensions
{
services.Configure<PostgresOptions>(sectionName, configuration.GetSection(sectionName));
services.AddSingleton<ConcelierDataSource>();
services.AddStartupMigrations(
ConcelierDataSource.DefaultSchemaName,
"Concelier.Storage",
typeof(ConcelierDataSource).Assembly);
// Register repositories
services.AddScoped<IAdvisoryRepository, AdvisoryRepository>();
services.AddScoped<IPostgresAdvisoryStore, PostgresAdvisoryStore>();
services.AddScoped<IAdvisoryCanonicalRepository, AdvisoryCanonicalRepository>();
services.AddScoped<ICanonicalAdvisoryStore, PostgresCanonicalAdvisoryStore>();
services.AddScoped<IMergeHashCalculator, MergeHashCalculatorAdapter>();
services.AddScoped<StellaOps.Concelier.Merge.Identity.IMergeHashCalculator, StellaOps.Concelier.Merge.Identity.MergeHashCalculator>();
services.AddScoped<ISourceRepository, SourceRepository>();
services.AddScoped<IAdvisorySourceReadRepository, AdvisorySourceReadRepository>();
services.AddScoped<IAdvisoryAliasRepository, AdvisoryAliasRepository>();
@@ -86,10 +96,18 @@ public static class ServiceCollectionExtensions
{
services.Configure(configureOptions);
services.AddSingleton<ConcelierDataSource>();
services.AddStartupMigrations(
ConcelierDataSource.DefaultSchemaName,
"Concelier.Storage",
typeof(ConcelierDataSource).Assembly);
// Register repositories
services.AddScoped<IAdvisoryRepository, AdvisoryRepository>();
services.AddScoped<IPostgresAdvisoryStore, PostgresAdvisoryStore>();
services.AddScoped<IAdvisoryCanonicalRepository, AdvisoryCanonicalRepository>();
services.AddScoped<ICanonicalAdvisoryStore, PostgresCanonicalAdvisoryStore>();
services.AddScoped<IMergeHashCalculator, MergeHashCalculatorAdapter>();
services.AddScoped<StellaOps.Concelier.Merge.Identity.IMergeHashCalculator, StellaOps.Concelier.Merge.Identity.MergeHashCalculator>();
services.AddScoped<ISourceRepository, SourceRepository>();
services.AddScoped<IAdvisorySourceReadRepository, AdvisorySourceReadRepository>();
services.AddScoped<IAdvisoryAliasRepository, AdvisoryAliasRepository>();