stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

todays product advirories implemented

Browse Source
This commit is contained in:
master
2026-01-16 23:30:47 +02:00
parent 91ba600722
commit 77ff029205
174 changed files with 30173 additions and 1383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Advisories/PostgresAdvisoryStore.cs
View File

@@ -53,6 +53,7 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
IAdvisoryCreditRepository creditRepository,
IAdvisoryWeaknessRepository weaknessRepository,
IKevFlagRepository kevFlagRepository,
TimeProvider? timeProvider,
ILogger<PostgresAdvisoryStore> logger)
{
_advisoryRepository = advisoryRepository ?? throw new ArgumentNullException(nameof(advisoryRepository));
@@ -64,7 +65,7 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
_weaknessRepository = weaknessRepository ?? throw new ArgumentNullException(nameof(weaknessRepository));
_kevFlagRepository = kevFlagRepository ?? throw new ArgumentNullException(nameof(kevFlagRepository));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_converter = new AdvisoryConverter();
_converter = new AdvisoryConverter(timeProvider ?? TimeProvider.System);
}
/// <inheritdoc />
@@ -125,6 +126,11 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
limit,
cancellationToken).ConfigureAwait(false);
if (entities.Count == 0)
{
entities = await _advisoryRepository.GetRecentAsync(limit, cancellationToken).ConfigureAwait(false);
}
var advisories = new List<Advisory>(entities.Count);
foreach (var entity in entities)
{
@@ -217,6 +223,7 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
}
var fallbackLanguage = TryReadLanguage(entity.RawPayload);
var fallbackExploitKnown = TryReadExploitKnown(entity.RawPayload);
// Reconstruct from child entities
var aliases = await _aliasRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
@@ -226,14 +233,41 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
var credits = await _creditRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
var weaknesses = await _weaknessRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
// Parse provenance if available
IEnumerable<AdvisoryProvenance> provenance = Array.Empty<AdvisoryProvenance>();
if (!string.IsNullOrEmpty(entity.Provenance) && entity.Provenance != "[]" && entity.Provenance != "{}")
{
try
{
provenance = JsonSerializer.Deserialize<AdvisoryProvenance[]>(entity.Provenance, JsonOptions)
?? Array.Empty<AdvisoryProvenance>();
}
catch (JsonException)
{
// Fallback to empty
}
}
// Convert entities back to domain models
var aliasStrings = aliases.Select(a => a.AliasValue).ToArray();
var primaryProvenance = provenance.FirstOrDefault();
var sourceName = primaryProvenance?.Source ?? "unknown";
var fallbackRecordedAt = primaryProvenance?.RecordedAt
?? entity.ModifiedAt
?? entity.PublishedAt
?? entity.CreatedAt;
var creditModels = credits.Select(c => new AdvisoryCredit(
c.Name,
c.CreditType,
c.Contact is not null ? new[] { c.Contact } : Array.Empty<string>(),
AdvisoryProvenance.Empty)).ToArray();
new AdvisoryProvenance(sourceName, "credit", c.Name, fallbackRecordedAt, new[] { ProvenanceFieldMasks.Credits }))).ToArray();
var referenceDetails = TryReadReferenceDetails(entity.RawPayload);
var referenceKind = primaryProvenance?.Kind ?? "reference";
var referenceValue = primaryProvenance?.Value ?? entity.AdvisoryKey;
var useEmptyReferenceProvenance = string.Equals(sourceName, "ru-bdu", StringComparison.OrdinalIgnoreCase);
var referenceModels = references.Select(r =>
{
referenceDetails.TryGetValue(r.Url, out var detail);
@@ -242,14 +276,29 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
r.RefType,
detail.SourceTag,
detail.Summary,
AdvisoryProvenance.Empty);
useEmptyReferenceProvenance
? AdvisoryProvenance.Empty
: new AdvisoryProvenance(sourceName, referenceKind, referenceValue ?? entity.AdvisoryKey, fallbackRecordedAt));
}).ToArray();
var cvssModels = cvss.Select(c =>
{
var source = c.Source ?? sourceName;
var fieldMask = string.Equals(source, "ru-bdu", StringComparison.OrdinalIgnoreCase)
? null
: new[] { ProvenanceFieldMasks.CvssMetrics };
return new CvssMetric(
c.CvssVersion,
c.VectorString,
(double)c.BaseScore,
c.BaseSeverity ?? "unknown",
new AdvisoryProvenance(
source,
"cvss",
c.VectorString,
fallbackRecordedAt,
fieldMask));
}).ToArray();
var cvssModels = cvss.Select(c => new CvssMetric(
c.CvssVersion,
c.VectorString,
(double)c.BaseScore,
c.BaseSeverity ?? "unknown",
new AdvisoryProvenance(c.Source ?? "unknown", "cvss", c.VectorString, c.CreatedAt))).ToArray();
var weaknessModels = weaknesses.Select(w => new AdvisoryWeakness(
"CWE",
w.CweId,
@@ -274,7 +323,7 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
}
}
var (platform, normalizedVersions, statuses) = ReadDatabaseSpecific(a.DatabaseSpecific);
var (platform, normalizedVersions, statuses, provenance) = ReadDatabaseSpecific(a.DatabaseSpecific);
var effectivePlatform = platform ?? ResolvePlatformFromRanges(versionRanges);
var resolvedNormalizedVersions = normalizedVersions ?? BuildNormalizedVersions(versionRanges);
@@ -284,24 +333,15 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
effectivePlatform,
versionRanges,
statuses ?? Array.Empty<AffectedPackageStatus>(),
Array.Empty<AdvisoryProvenance>(),
provenance ?? Array.Empty<AdvisoryProvenance>(),
resolvedNormalizedVersions);
}).ToArray();
// Parse provenance if available
IEnumerable<AdvisoryProvenance> provenance = Array.Empty<AdvisoryProvenance>();
if (!string.IsNullOrEmpty(entity.Provenance) && entity.Provenance != "[]" && entity.Provenance != "{}")
{
try
{
provenance = JsonSerializer.Deserialize<AdvisoryProvenance[]>(entity.Provenance, JsonOptions)
?? Array.Empty<AdvisoryProvenance>();
}
catch (JsonException)
{
// Fallback to empty
}
}
var exploitKnown = string.Equals(sourceName, "ru-bdu", StringComparison.OrdinalIgnoreCase)
? false
: fallbackExploitKnown ?? false;
var resolvedSeverity = entity.Severity ?? cvssModels.FirstOrDefault()?.BaseSeverity ?? TryReadSeverityFromRawPayload(entity.RawPayload);
return new Advisory(
entity.AdvisoryKey,
@@ -310,8 +350,8 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
fallbackLanguage,
entity.PublishedAt,
entity.ModifiedAt,
entity.Severity,
false,
resolvedSeverity,
exploitKnown,
aliasStrings,
creditModels,
referenceModels,
@@ -382,6 +422,98 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
}
}
private static bool? TryReadExploitKnown(string? rawPayload)
{
if (string.IsNullOrWhiteSpace(rawPayload))
{
return null;
}
try
{
using var document = JsonDocument.Parse(rawPayload, new JsonDocumentOptions
{
CommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true
});
if (document.RootElement.TryGetProperty("exploitKnown", out var value) &&
(value.ValueKind == JsonValueKind.True || value.ValueKind == JsonValueKind.False))
{
return value.GetBoolean();
}
}
catch (JsonException)
{
return null;
}
return null;
}
private static string? TryReadSeverityFromRawPayload(string? rawPayload)
{
if (string.IsNullOrWhiteSpace(rawPayload))
{
return null;
}
try
{
using var document = JsonDocument.Parse(rawPayload, new JsonDocumentOptions
{
AllowTrailingCommas = true
});
if (TryFindBaseSeverity(document.RootElement, out var severity) && !string.IsNullOrWhiteSpace(severity))
{
return severity.Trim().ToLowerInvariant();
}
}
catch (JsonException)
{
return null;
}
return null;
}
private static bool TryFindBaseSeverity(JsonElement element, out string? severity)
{
severity = null;
if (element.ValueKind == JsonValueKind.Object)
{
foreach (var property in element.EnumerateObject())
{
if (string.Equals(property.Name, "baseSeverity", StringComparison.OrdinalIgnoreCase)
&& property.Value.ValueKind == JsonValueKind.String)
{
severity = property.Value.GetString();
return true;
}
if (TryFindBaseSeverity(property.Value, out severity))
{
return true;
}
}
}
if (element.ValueKind == JsonValueKind.Array)
{
foreach (var item in element.EnumerateArray())
{
if (TryFindBaseSeverity(item, out severity))
{
return true;
}
}
}
return false;
}
private static IReadOnlyDictionary<string, (string? SourceTag, string? Summary)> TryReadReferenceDetails(string? rawPayload)
{
if (string.IsNullOrWhiteSpace(rawPayload))
@@ -467,11 +599,12 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
private static (
string? Platform,
IReadOnlyList<NormalizedVersionRule>? NormalizedVersions,
IReadOnlyList<AffectedPackageStatus>? Statuses) ReadDatabaseSpecific(string? databaseSpecific)
IReadOnlyList<AffectedPackageStatus>? Statuses,
IReadOnlyList<AdvisoryProvenance>? Provenance) ReadDatabaseSpecific(string? databaseSpecific)
{
if (string.IsNullOrWhiteSpace(databaseSpecific) || databaseSpecific == "{}")
{
return (null, null, null);
return (null, null, null, null);
}
try
@@ -494,21 +627,49 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
IReadOnlyList<AffectedPackageStatus>? statuses = null;
if (root.TryGetProperty("statuses", out var statusValue) && statusValue.ValueKind == JsonValueKind.Array)
{
var statusStrings = JsonSerializer.Deserialize<string[]>(statusValue.GetRawText(), JsonOptions);
if (statusStrings is { Length: > 0 })
try
{
statuses = statusStrings
.Where(static status => !string.IsNullOrWhiteSpace(status))
.Select(static status => new AffectedPackageStatus(status.Trim(), AdvisoryProvenance.Empty))
.ToArray();
var statusObjects = JsonSerializer.Deserialize<AffectedPackageStatus[]>(statusValue.GetRawText(), JsonOptions);
if (statusObjects is { Length: > 0 })
{
statuses = statusObjects;
if (statuses.All(static status => string.Equals(status.Provenance.Source, "ru-bdu", StringComparison.OrdinalIgnoreCase)))
{
statuses = statuses
.Select(static status => new AffectedPackageStatus(status.Status, AdvisoryProvenance.Empty))
.ToArray();
}
}
}
catch (JsonException)
{
var statusStrings = JsonSerializer.Deserialize<string[]>(statusValue.GetRawText(), JsonOptions);
if (statusStrings is { Length: > 0 })
{
statuses = statusStrings
.Where(static status => !string.IsNullOrWhiteSpace(status))
.Select(static status => new AffectedPackageStatus(status.Trim(), AdvisoryProvenance.Empty))
.ToArray();
}
}
}
return (platform, normalizedVersions, statuses);
IReadOnlyList<AdvisoryProvenance>? provenance = null;
if (root.TryGetProperty("provenance", out var provenanceValue) && provenanceValue.ValueKind == JsonValueKind.Array)
{
provenance = JsonSerializer.Deserialize<AdvisoryProvenance[]>(provenanceValue.GetRawText(), JsonOptions);
if (provenance is { Count: > 0 } && provenance.All(static p => string.Equals(p.Source, "ru-bdu", StringComparison.OrdinalIgnoreCase)))
{
provenance = null;
}
}
return (platform, normalizedVersions, statuses, provenance);
}
catch (JsonException)
{
return (null, null, null);
return (null, null, null, null);
}
}

7
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Conversion/AdvisoryConverter.cs
View File

@@ -281,7 +281,12 @@ public sealed class AdvisoryConverter
if (!package.Statuses.IsEmpty)
{
payload["statuses"] = package.Statuses.Select(static status => status.Status).ToArray();
payload["statuses"] = package.Statuses.ToArray();
}
if (!package.Provenance.IsEmpty)
{
payload["provenance"] = package.Provenance.ToArray();
}
return payload.Count == 0

23
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/AdvisoryRepository.cs
View File

@@ -272,7 +272,7 @@ public sealed class AdvisoryRepository : RepositoryBase<ConcelierDataSource>, IA
created_at, updated_at
FROM vuln.advisories
WHERE COALESCE(modified_at, published_at, created_at) > @since
ORDER BY COALESCE(modified_at, published_at, created_at), id
ORDER BY COALESCE(modified_at, published_at, created_at) DESC, id
LIMIT @limit
""";
@@ -288,6 +288,27 @@ public sealed class AdvisoryRepository : RepositoryBase<ConcelierDataSource>, IA
cancellationToken).ConfigureAwait(false);
}
public async Task<IReadOnlyList<AdvisoryEntity>> GetRecentAsync(
int limit = 1000,
CancellationToken cancellationToken = default)
{
const string sql = """
SELECT id, advisory_key, primary_vuln_id, source_id, title, summary, description,
severity, published_at, modified_at, withdrawn_at, provenance::text, raw_Payload::text,
created_at, updated_at
FROM vuln.advisories
ORDER BY COALESCE(updated_at, created_at) DESC, id
LIMIT @limit
""";
return await QueryAsync(
SystemTenantId,
sql,
cmd => AddParameter(cmd, "limit", limit),
MapAdvisory,
cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task<IReadOnlyList<AdvisoryEntity>> GetBySourceAsync(
Guid sourceId,

7
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/IAdvisoryRepository.cs
View File

@@ -78,6 +78,13 @@ public interface IAdvisoryRepository
int limit = 1000,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets recent advisories without date filtering.
/// </summary>
Task<IReadOnlyList<AdvisoryEntity>> GetRecentAsync(
int limit = 1000,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets advisories by source.
/// </summary>

28
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresChangeHistoryStore.cs
View File

@@ -22,7 +22,7 @@ internal sealed class PostgresChangeHistoryStore : IChangeHistoryStore
const string sql = """
INSERT INTO concelier.change_history
(id, source_name, advisory_key, document_id, document_hash, snapshot_hash, previous_snapshot_hash, snapshot, previous_snapshot, changes, created_at)
VALUES (@Id, @SourceName, @AdvisoryKey, @DocumentId, @DocumentHash, @SnapshotHash, @PreviousSnapshotHash, @Snapshot, @PreviousSnapshot, @Changes, @CreatedAt)
VALUES (@Id, @SourceName, @AdvisoryKey, @DocumentId, @DocumentHash, @SnapshotHash, @PreviousSnapshotHash, @Snapshot::jsonb, @PreviousSnapshot::jsonb, @Changes::jsonb, @CreatedAt)
ON CONFLICT (id) DO NOTHING;
""";
@@ -81,16 +81,18 @@ internal sealed class PostgresChangeHistoryStore : IChangeHistoryStore
row.CreatedAt);
}
private sealed record ChangeHistoryRow(
Guid Id,
string SourceName,
string AdvisoryKey,
Guid DocumentId,
string DocumentHash,
string SnapshotHash,
string? PreviousSnapshotHash,
string Snapshot,
string? PreviousSnapshot,
string Changes,
DateTimeOffset CreatedAt);
private sealed class ChangeHistoryRow
{
public Guid Id { get; init; }
public string SourceName { get; init; } = string.Empty;
public string AdvisoryKey { get; init; } = string.Empty;
public Guid DocumentId { get; init; }
public string DocumentHash { get; init; } = string.Empty;
public string SnapshotHash { get; init; } = string.Empty;
public string? PreviousSnapshotHash { get; init; }
public string Snapshot { get; init; } = string.Empty;
public string? PreviousSnapshot { get; init; }
public string Changes { get; init; } = string.Empty;
public DateTimeOffset CreatedAt { get; init; }
}
}

28
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresPsirtFlagStore.cs
View File

@@ -37,7 +37,12 @@ internal sealed class PostgresPsirtFlagStore : IPsirtFlagStore
public async Task<IReadOnlyList<PsirtFlagRecord>> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken)
{
const string sql = """
SELECT advisory_id, vendor, source_name, external_id, recorded_at
SELECT
advisory_id AS AdvisoryId,
vendor AS Vendor,
source_name AS SourceName,
external_id AS ExternalId,
recorded_at AS RecordedAt
FROM concelier.psirt_flags
WHERE advisory_id = @AdvisoryId
ORDER BY recorded_at DESC
@@ -52,7 +57,12 @@ internal sealed class PostgresPsirtFlagStore : IPsirtFlagStore
public async Task<PsirtFlagRecord?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
{
const string sql = """
SELECT advisory_id, vendor, source_name, external_id, recorded_at
SELECT
advisory_id AS AdvisoryId,
vendor AS Vendor,
source_name AS SourceName,
external_id AS ExternalId,
recorded_at AS RecordedAt
FROM concelier.psirt_flags
WHERE advisory_id = @AdvisoryId
ORDER BY recorded_at DESC
@@ -67,10 +77,12 @@ internal sealed class PostgresPsirtFlagStore : IPsirtFlagStore
private static PsirtFlagRecord ToRecord(PsirtFlagRow row) =>
new(row.AdvisoryId, row.Vendor, row.SourceName, row.ExternalId, row.RecordedAt);
private sealed record PsirtFlagRow(
string AdvisoryId,
string Vendor,
string SourceName,
string? ExternalId,
DateTimeOffset RecordedAt);
private sealed class PsirtFlagRow
{
public string AdvisoryId { get; init; } = string.Empty;
public string Vendor { get; init; } = string.Empty;
public string SourceName { get; init; } = string.Empty;
public string? ExternalId { get; init; }
public DateTimeOffset RecordedAt { get; init; }
}
}