Add integration tests for migration categories and execution
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled

- Implemented MigrationCategoryTests to validate migration categorization for startup, release, seed, and data migrations.
- Added tests for edge cases, including null, empty, and whitespace migration names.
- Created StartupMigrationHostTests to verify the behavior of the migration host with real PostgreSQL instances using Testcontainers.
- Included tests for migration execution, schema creation, and handling of pending release migrations.
- Added SQL migration files for testing: creating a test table, adding a column, a release migration, and seeding data.
This commit is contained in:
master
2025-12-04 19:10:54 +02:00
parent 600f3a7a3c
commit 75f6942769
301 changed files with 32810 additions and 1128 deletions

View File

@@ -0,0 +1,51 @@
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Storage.Postgres.Advisories;
/// <summary>
/// PostgreSQL advisory storage interface.
/// This interface mirrors the MongoDB IAdvisoryStore but without MongoDB-specific parameters.
/// </summary>
/// <remarks>
/// Used by connectors when configured to write to PostgreSQL storage.
/// </remarks>
public interface IPostgresAdvisoryStore
{
/// <summary>
/// Upserts an advisory and all its child entities.
/// </summary>
/// <param name="advisory">The advisory domain model to store.</param>
/// <param name="sourceId">Optional source ID to associate with the advisory.</param>
/// <param name="cancellationToken">Cancellation token.</param>
Task UpsertAsync(Advisory advisory, Guid? sourceId, CancellationToken cancellationToken);
/// <summary>
/// Finds an advisory by its key.
/// </summary>
/// <param name="advisoryKey">The advisory key.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The advisory if found, null otherwise.</returns>
Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken);
/// <summary>
/// Gets the most recently modified advisories.
/// </summary>
/// <param name="limit">Maximum number of advisories to return.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>List of recent advisories.</returns>
Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken);
/// <summary>
/// Streams all advisories for bulk operations.
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Async enumerable of advisories.</returns>
IAsyncEnumerable<Advisory> StreamAsync(CancellationToken cancellationToken);
/// <summary>
/// Gets the count of advisories in the store.
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Total count of advisories.</returns>
Task<long> CountAsync(CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,301 @@
using System.Runtime.CompilerServices;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Postgres.Conversion;
using StellaOps.Concelier.Storage.Postgres.Models;
using StellaOps.Concelier.Storage.Postgres.Repositories;
namespace StellaOps.Concelier.Storage.Postgres.Advisories;
/// <summary>
/// PostgreSQL implementation of advisory storage.
/// Uses the AdvisoryConverter to transform domain models to relational entities
/// and the AdvisoryRepository to persist them.
/// </summary>
/// <remarks>
/// Tasks: PG-T5b.2.1, PG-T5b.2.2, PG-T5b.2.3 - Enables importers to write to PostgreSQL.
/// </remarks>
public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore
{
private readonly IAdvisoryRepository _advisoryRepository;
private readonly IAdvisoryAliasRepository _aliasRepository;
private readonly IAdvisoryCvssRepository _cvssRepository;
private readonly IAdvisoryAffectedRepository _affectedRepository;
private readonly IAdvisoryReferenceRepository _referenceRepository;
private readonly IAdvisoryCreditRepository _creditRepository;
private readonly IAdvisoryWeaknessRepository _weaknessRepository;
private readonly IKevFlagRepository _kevFlagRepository;
private readonly AdvisoryConverter _converter;
private readonly ILogger<PostgresAdvisoryStore> _logger;
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false
};
public PostgresAdvisoryStore(
IAdvisoryRepository advisoryRepository,
IAdvisoryAliasRepository aliasRepository,
IAdvisoryCvssRepository cvssRepository,
IAdvisoryAffectedRepository affectedRepository,
IAdvisoryReferenceRepository referenceRepository,
IAdvisoryCreditRepository creditRepository,
IAdvisoryWeaknessRepository weaknessRepository,
IKevFlagRepository kevFlagRepository,
ILogger<PostgresAdvisoryStore> logger)
{
_advisoryRepository = advisoryRepository ?? throw new ArgumentNullException(nameof(advisoryRepository));
_aliasRepository = aliasRepository ?? throw new ArgumentNullException(nameof(aliasRepository));
_cvssRepository = cvssRepository ?? throw new ArgumentNullException(nameof(cvssRepository));
_affectedRepository = affectedRepository ?? throw new ArgumentNullException(nameof(affectedRepository));
_referenceRepository = referenceRepository ?? throw new ArgumentNullException(nameof(referenceRepository));
_creditRepository = creditRepository ?? throw new ArgumentNullException(nameof(creditRepository));
_weaknessRepository = weaknessRepository ?? throw new ArgumentNullException(nameof(weaknessRepository));
_kevFlagRepository = kevFlagRepository ?? throw new ArgumentNullException(nameof(kevFlagRepository));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_converter = new AdvisoryConverter();
}
/// <inheritdoc />
public async Task UpsertAsync(Advisory advisory, Guid? sourceId, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(advisory);
_logger.LogDebug("Upserting advisory {AdvisoryKey} to PostgreSQL", advisory.AdvisoryKey);
// Convert domain model to PostgreSQL entities
var result = _converter.ConvertFromDomain(advisory, sourceId);
// Use the repository's atomic upsert which handles all child tables in a transaction
await _advisoryRepository.UpsertAsync(
result.Advisory,
result.Aliases,
result.Cvss,
result.Affected,
result.References,
result.Credits,
result.Weaknesses,
result.KevFlags,
cancellationToken).ConfigureAwait(false);
_logger.LogDebug(
"Upserted advisory {AdvisoryKey} with {ChildCount} child entities",
advisory.AdvisoryKey,
result.TotalChildEntities);
}
/// <inheritdoc />
public async Task<Advisory?> FindAsync(string advisoryKey, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrEmpty(advisoryKey);
var entity = await _advisoryRepository.GetByKeyAsync(advisoryKey, cancellationToken).ConfigureAwait(false);
if (entity is null)
{
return null;
}
return await ReconstructAdvisoryAsync(entity, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task<IReadOnlyList<Advisory>> GetRecentAsync(int limit, CancellationToken cancellationToken)
{
var entities = await _advisoryRepository.GetModifiedSinceAsync(
DateTimeOffset.MinValue,
limit,
cancellationToken).ConfigureAwait(false);
var advisories = new List<Advisory>(entities.Count);
foreach (var entity in entities)
{
var advisory = await ReconstructAdvisoryAsync(entity, cancellationToken).ConfigureAwait(false);
advisories.Add(advisory);
}
return advisories;
}
/// <inheritdoc />
public async IAsyncEnumerable<Advisory> StreamAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
var offset = 0;
const int batchSize = 100;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
var entities = await _advisoryRepository.GetModifiedSinceAsync(
DateTimeOffset.MinValue,
batchSize,
cancellationToken).ConfigureAwait(false);
if (entities.Count == 0)
{
break;
}
foreach (var entity in entities)
{
cancellationToken.ThrowIfCancellationRequested();
yield return await ReconstructAdvisoryAsync(entity, cancellationToken).ConfigureAwait(false);
}
if (entities.Count < batchSize)
{
break;
}
offset += batchSize;
}
}
/// <inheritdoc />
public Task<long> CountAsync(CancellationToken cancellationToken)
{
return _advisoryRepository.CountAsync(cancellationToken);
}
/// <summary>
/// Reconstructs an Advisory domain model from a PostgreSQL entity.
/// </summary>
private async Task<Advisory> ReconstructAdvisoryAsync(AdvisoryEntity entity, CancellationToken cancellationToken)
{
// If raw payload is available, deserialize from it for full fidelity
if (!string.IsNullOrEmpty(entity.RawPayload))
{
try
{
var advisory = JsonSerializer.Deserialize<Advisory>(entity.RawPayload, JsonOptions);
if (advisory is not null)
{
return advisory;
}
}
catch (JsonException ex)
{
_logger.LogWarning(ex, "Failed to deserialize raw payload for advisory {AdvisoryKey}, reconstructing from entities", entity.AdvisoryKey);
}
}
// Reconstruct from child entities
var aliases = await _aliasRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
var cvss = await _cvssRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
var affected = await _affectedRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
var references = await _referenceRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
var credits = await _creditRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
var weaknesses = await _weaknessRepository.GetByAdvisoryAsync(entity.Id, cancellationToken).ConfigureAwait(false);
// Convert entities back to domain models
var aliasStrings = aliases.Select(a => a.AliasValue).ToArray();
var creditModels = credits.Select(c => new AdvisoryCredit(
c.Name,
c.CreditType,
c.Contact is not null ? new[] { c.Contact } : Array.Empty<string>(),
AdvisoryProvenance.Empty)).ToArray();
var referenceModels = references.Select(r => new AdvisoryReference(
r.Url,
r.RefType,
null,
null,
AdvisoryProvenance.Empty)).ToArray();
var cvssModels = cvss.Select(c => new CvssMetric(
c.CvssVersion,
c.VectorString,
(double)c.BaseScore,
c.BaseSeverity ?? "unknown",
new AdvisoryProvenance(c.Source ?? "unknown", "cvss", c.VectorString, c.CreatedAt))).ToArray();
var weaknessModels = weaknesses.Select(w => new AdvisoryWeakness(
"CWE",
w.CweId,
w.Description,
null,
w.Source is not null ? new[] { new AdvisoryProvenance(w.Source, "cwe", w.CweId, w.CreatedAt) } : Array.Empty<AdvisoryProvenance>())).ToArray();
// Convert affected packages
var affectedModels = affected.Select(a =>
{
IEnumerable<AffectedVersionRange> versionRanges = Array.Empty<AffectedVersionRange>();
if (!string.IsNullOrEmpty(a.VersionRange) && a.VersionRange != "{}")
{
try
{
versionRanges = JsonSerializer.Deserialize<AffectedVersionRange[]>(a.VersionRange, JsonOptions)
?? Array.Empty<AffectedVersionRange>();
}
catch (JsonException)
{
// Fallback to empty
}
}
return new AffectedPackage(
MapEcosystemToType(a.Ecosystem),
a.PackageName,
null,
versionRanges);
}).ToArray();
// Parse provenance if available
IEnumerable<AdvisoryProvenance> provenance = Array.Empty<AdvisoryProvenance>();
if (!string.IsNullOrEmpty(entity.Provenance) && entity.Provenance != "[]" && entity.Provenance != "{}")
{
try
{
provenance = JsonSerializer.Deserialize<AdvisoryProvenance[]>(entity.Provenance, JsonOptions)
?? Array.Empty<AdvisoryProvenance>();
}
catch (JsonException)
{
// Fallback to empty
}
}
return new Advisory(
entity.AdvisoryKey,
entity.Title ?? entity.AdvisoryKey,
entity.Summary,
null,
entity.PublishedAt,
entity.ModifiedAt,
entity.Severity,
false,
aliasStrings,
creditModels,
referenceModels,
affectedModels,
cvssModels,
provenance,
entity.Description,
weaknessModels,
null);
}
private static string MapEcosystemToType(string ecosystem)
{
return ecosystem.ToLowerInvariant() switch
{
"npm" => "semver",
"pypi" => "semver",
"maven" => "semver",
"nuget" => "semver",
"cargo" => "semver",
"go" => "semver",
"rubygems" => "semver",
"composer" => "semver",
"hex" => "semver",
"pub" => "semver",
"rpm" => "rpm",
"deb" => "deb",
"apk" => "semver",
"cpe" => "cpe",
"vendor" => "vendor",
"ics" => "ics-vendor",
"generic" => "semver",
_ => "semver"
};
}
}

View File

@@ -0,0 +1,56 @@
using StellaOps.Concelier.Storage.Postgres.Models;
namespace StellaOps.Concelier.Storage.Postgres.Conversion;
/// <summary>
/// Result of converting a MongoDB advisory document to PostgreSQL entities.
/// Contains the main advisory entity and all related child entities.
/// </summary>
public sealed class AdvisoryConversionResult
{
/// <summary>
/// The main advisory entity.
/// </summary>
public required AdvisoryEntity Advisory { get; init; }
/// <summary>
/// Alias entities (CVE, GHSA, etc.).
/// </summary>
public IReadOnlyList<AdvisoryAliasEntity> Aliases { get; init; } = Array.Empty<AdvisoryAliasEntity>();
/// <summary>
/// CVSS score entities.
/// </summary>
public IReadOnlyList<AdvisoryCvssEntity> Cvss { get; init; } = Array.Empty<AdvisoryCvssEntity>();
/// <summary>
/// Affected package entities.
/// </summary>
public IReadOnlyList<AdvisoryAffectedEntity> Affected { get; init; } = Array.Empty<AdvisoryAffectedEntity>();
/// <summary>
/// Reference URL entities.
/// </summary>
public IReadOnlyList<AdvisoryReferenceEntity> References { get; init; } = Array.Empty<AdvisoryReferenceEntity>();
/// <summary>
/// Credit entities.
/// </summary>
public IReadOnlyList<AdvisoryCreditEntity> Credits { get; init; } = Array.Empty<AdvisoryCreditEntity>();
/// <summary>
/// Weakness (CWE) entities.
/// </summary>
public IReadOnlyList<AdvisoryWeaknessEntity> Weaknesses { get; init; } = Array.Empty<AdvisoryWeaknessEntity>();
/// <summary>
/// Known Exploited Vulnerabilities (KEV) flag entities.
/// </summary>
public IReadOnlyList<KevFlagEntity> KevFlags { get; init; } = Array.Empty<KevFlagEntity>();
/// <summary>
/// Total number of child entities.
/// </summary>
public int TotalChildEntities =>
Aliases.Count + Cvss.Count + Affected.Count + References.Count + Credits.Count + Weaknesses.Count + KevFlags.Count;
}

View File

@@ -0,0 +1,659 @@
using System.Text.Json;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Storage.Postgres.Models;
namespace StellaOps.Concelier.Storage.Postgres.Conversion;
/// <summary>
/// Converts MongoDB advisory documents to PostgreSQL entity structures.
/// This converter handles the transformation from MongoDB's document-based storage
/// to PostgreSQL's relational structure with normalized child tables.
/// </summary>
/// <remarks>
/// Task: PG-T5b.1.1 - Build AdvisoryConverter to parse MongoDB documents
/// Task: PG-T5b.1.2 - Map to relational structure with child tables
/// Task: PG-T5b.1.3 - Preserve provenance JSONB
/// Task: PG-T5b.1.4 - Handle version ranges (keep as JSONB)
/// </remarks>
public sealed class AdvisoryConverter
{
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false
};
/// <summary>
/// Converts a MongoDB BsonDocument payload to PostgreSQL entities.
/// </summary>
/// <param name="payload">The MongoDB advisory payload (BsonDocument).</param>
/// <param name="sourceId">Optional source ID to associate with the advisory.</param>
/// <returns>A conversion result containing the main entity and all child entities.</returns>
public AdvisoryConversionResult Convert(BsonDocument payload, Guid? sourceId = null)
{
ArgumentNullException.ThrowIfNull(payload);
var advisoryKey = payload.GetValue("advisoryKey", defaultValue: null)?.AsString
?? throw new InvalidOperationException("advisoryKey missing from payload.");
var title = payload.GetValue("title", defaultValue: null)?.AsString ?? advisoryKey;
var summary = TryGetString(payload, "summary");
var description = TryGetString(payload, "description");
var severity = TryGetString(payload, "severity");
var published = TryReadDateTime(payload, "published");
var modified = TryReadDateTime(payload, "modified");
// Extract primary vulnerability ID from aliases (first CVE if available)
var aliases = ExtractAliases(payload);
var cveAlias = aliases.FirstOrDefault(a => a.AliasType == "cve");
var firstAlias = aliases.FirstOrDefault();
var primaryVulnId = cveAlias != default ? cveAlias.AliasValue
: (firstAlias != default ? firstAlias.AliasValue : advisoryKey);
// Extract provenance and serialize to JSONB
var provenanceJson = ExtractProvenanceJson(payload);
// Create the main advisory entity
var advisoryId = Guid.NewGuid();
var now = DateTimeOffset.UtcNow;
var advisory = new AdvisoryEntity
{
Id = advisoryId,
AdvisoryKey = advisoryKey,
PrimaryVulnId = primaryVulnId,
SourceId = sourceId,
Title = title,
Summary = summary,
Description = description,
Severity = severity,
PublishedAt = published,
ModifiedAt = modified,
WithdrawnAt = null,
Provenance = provenanceJson,
RawPayload = payload.ToJson(),
CreatedAt = now,
UpdatedAt = now
};
// Convert all child entities
var aliasEntities = ConvertAliases(advisoryId, aliases, now);
var cvssEntities = ConvertCvss(advisoryId, payload, now);
var affectedEntities = ConvertAffected(advisoryId, payload, now);
var referenceEntities = ConvertReferences(advisoryId, payload, now);
var creditEntities = ConvertCredits(advisoryId, payload, now);
var weaknessEntities = ConvertWeaknesses(advisoryId, payload, now);
var kevFlags = ConvertKevFlags(advisoryId, payload, now);
return new AdvisoryConversionResult
{
Advisory = advisory,
Aliases = aliasEntities,
Cvss = cvssEntities,
Affected = affectedEntities,
References = referenceEntities,
Credits = creditEntities,
Weaknesses = weaknessEntities,
KevFlags = kevFlags
};
}
/// <summary>
/// Converts an Advisory domain model directly to PostgreSQL entities.
/// </summary>
/// <param name="advisory">The Advisory domain model.</param>
/// <param name="sourceId">Optional source ID.</param>
/// <returns>A conversion result containing all entities.</returns>
public AdvisoryConversionResult ConvertFromDomain(Advisory advisory, Guid? sourceId = null)
{
ArgumentNullException.ThrowIfNull(advisory);
var advisoryId = Guid.NewGuid();
var now = DateTimeOffset.UtcNow;
// Determine primary vulnerability ID
var primaryVulnId = advisory.Aliases
.FirstOrDefault(a => a.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
?? advisory.Aliases.FirstOrDefault()
?? advisory.AdvisoryKey;
// Serialize provenance to JSON
var provenanceJson = JsonSerializer.Serialize(advisory.Provenance, JsonOptions);
var entity = new AdvisoryEntity
{
Id = advisoryId,
AdvisoryKey = advisory.AdvisoryKey,
PrimaryVulnId = primaryVulnId,
SourceId = sourceId,
Title = advisory.Title,
Summary = advisory.Summary,
Description = advisory.Description,
Severity = advisory.Severity,
PublishedAt = advisory.Published,
ModifiedAt = advisory.Modified,
WithdrawnAt = null,
Provenance = provenanceJson,
RawPayload = CanonicalJsonSerializer.Serialize(advisory),
CreatedAt = now,
UpdatedAt = now
};
// Convert aliases
var aliasEntities = new List<AdvisoryAliasEntity>();
var isPrimarySet = false;
foreach (var alias in advisory.Aliases)
{
var aliasType = DetermineAliasType(alias);
var isPrimary = !isPrimarySet && aliasType == "cve";
if (isPrimary) isPrimarySet = true;
aliasEntities.Add(new AdvisoryAliasEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
AliasType = aliasType,
AliasValue = alias,
IsPrimary = isPrimary,
CreatedAt = now
});
}
// Convert CVSS metrics
var cvssEntities = new List<AdvisoryCvssEntity>();
var isPrimaryCvss = true;
foreach (var metric in advisory.CvssMetrics)
{
cvssEntities.Add(new AdvisoryCvssEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
CvssVersion = metric.Version,
VectorString = metric.Vector,
BaseScore = (decimal)metric.BaseScore,
BaseSeverity = metric.BaseSeverity,
ExploitabilityScore = null,
ImpactScore = null,
Source = metric.Provenance.Source,
IsPrimary = isPrimaryCvss,
CreatedAt = now
});
isPrimaryCvss = false;
}
// Convert affected packages
var affectedEntities = new List<AdvisoryAffectedEntity>();
foreach (var pkg in advisory.AffectedPackages)
{
var ecosystem = MapTypeToEcosystem(pkg.Type);
var versionRangeJson = JsonSerializer.Serialize(pkg.VersionRanges, JsonOptions);
affectedEntities.Add(new AdvisoryAffectedEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
Ecosystem = ecosystem,
PackageName = pkg.Identifier,
Purl = BuildPurl(ecosystem, pkg.Identifier),
VersionRange = versionRangeJson,
VersionsAffected = null,
VersionsFixed = ExtractFixedVersions(pkg.VersionRanges),
DatabaseSpecific = null,
CreatedAt = now
});
}
// Convert references
var referenceEntities = new List<AdvisoryReferenceEntity>();
foreach (var reference in advisory.References)
{
referenceEntities.Add(new AdvisoryReferenceEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
RefType = reference.Kind ?? "web",
Url = reference.Url,
CreatedAt = now
});
}
// Convert credits
var creditEntities = new List<AdvisoryCreditEntity>();
foreach (var credit in advisory.Credits)
{
creditEntities.Add(new AdvisoryCreditEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
Name = credit.DisplayName,
Contact = credit.Contacts.FirstOrDefault(),
CreditType = credit.Role,
CreatedAt = now
});
}
// Convert weaknesses
var weaknessEntities = new List<AdvisoryWeaknessEntity>();
foreach (var weakness in advisory.Cwes)
{
weaknessEntities.Add(new AdvisoryWeaknessEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
CweId = weakness.Identifier,
Description = weakness.Name,
Source = weakness.Provenance.FirstOrDefault()?.Source,
CreatedAt = now
});
}
return new AdvisoryConversionResult
{
Advisory = entity,
Aliases = aliasEntities,
Cvss = cvssEntities,
Affected = affectedEntities,
References = referenceEntities,
Credits = creditEntities,
Weaknesses = weaknessEntities,
KevFlags = new List<KevFlagEntity>()
};
}
private static List<(string AliasType, string AliasValue, bool IsPrimary)> ExtractAliases(BsonDocument payload)
{
var result = new List<(string AliasType, string AliasValue, bool IsPrimary)>();
if (!payload.TryGetValue("aliases", out var aliasValue) || aliasValue is not BsonArray aliasArray)
{
return result;
}
var isPrimarySet = false;
foreach (var alias in aliasArray.OfType<BsonValue>().Where(x => x.IsString).Select(x => x.AsString))
{
var aliasType = DetermineAliasType(alias);
var isPrimary = !isPrimarySet && aliasType == "cve";
if (isPrimary) isPrimarySet = true;
result.Add((aliasType, alias, isPrimary));
}
return result;
}
private static string DetermineAliasType(string alias)
{
if (alias.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
return "cve";
if (alias.StartsWith("GHSA-", StringComparison.OrdinalIgnoreCase))
return "ghsa";
if (alias.StartsWith("RUSTSEC-", StringComparison.OrdinalIgnoreCase))
return "rustsec";
if (alias.StartsWith("GO-", StringComparison.OrdinalIgnoreCase))
return "go";
if (alias.StartsWith("PYSEC-", StringComparison.OrdinalIgnoreCase))
return "pysec";
if (alias.StartsWith("DSA-", StringComparison.OrdinalIgnoreCase))
return "dsa";
if (alias.StartsWith("RHSA-", StringComparison.OrdinalIgnoreCase))
return "rhsa";
if (alias.StartsWith("USN-", StringComparison.OrdinalIgnoreCase))
return "usn";
return "other";
}
private static string ExtractProvenanceJson(BsonDocument payload)
{
if (!payload.TryGetValue("provenance", out var provenanceValue) || provenanceValue is not BsonArray provenanceArray)
{
return "[]";
}
return provenanceArray.ToJson();
}
private static List<AdvisoryAliasEntity> ConvertAliases(
Guid advisoryId,
List<(string AliasType, string AliasValue, bool IsPrimary)> aliases,
DateTimeOffset now)
{
return aliases.Select(a => new AdvisoryAliasEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
AliasType = a.AliasType,
AliasValue = a.AliasValue,
IsPrimary = a.IsPrimary,
CreatedAt = now
}).ToList();
}
private static List<AdvisoryCvssEntity> ConvertCvss(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
{
var result = new List<AdvisoryCvssEntity>();
if (!payload.TryGetValue("cvssMetrics", out var cvssValue) || cvssValue is not BsonArray cvssArray)
{
return result;
}
var isPrimary = true;
foreach (var doc in cvssArray.OfType<BsonDocument>())
{
var version = doc.GetValue("version", defaultValue: null)?.AsString;
var vector = doc.GetValue("vector", defaultValue: null)?.AsString;
var baseScore = doc.TryGetValue("baseScore", out var scoreValue) && scoreValue.IsNumeric
? (decimal)scoreValue.ToDouble()
: 0m;
var baseSeverity = TryGetString(doc, "baseSeverity");
var source = doc.TryGetValue("provenance", out var provValue) && provValue.IsBsonDocument
? TryGetString(provValue.AsBsonDocument, "source")
: null;
if (string.IsNullOrEmpty(version) || string.IsNullOrEmpty(vector))
continue;
result.Add(new AdvisoryCvssEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
CvssVersion = version,
VectorString = vector,
BaseScore = baseScore,
BaseSeverity = baseSeverity,
ExploitabilityScore = null,
ImpactScore = null,
Source = source,
IsPrimary = isPrimary,
CreatedAt = now
});
isPrimary = false;
}
return result;
}
private static List<AdvisoryAffectedEntity> ConvertAffected(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
{
var result = new List<AdvisoryAffectedEntity>();
if (!payload.TryGetValue("affectedPackages", out var affectedValue) || affectedValue is not BsonArray affectedArray)
{
return result;
}
foreach (var doc in affectedArray.OfType<BsonDocument>())
{
var type = doc.GetValue("type", defaultValue: null)?.AsString ?? "semver";
var identifier = doc.GetValue("identifier", defaultValue: null)?.AsString;
if (string.IsNullOrEmpty(identifier))
continue;
var ecosystem = MapTypeToEcosystem(type);
// Version ranges kept as JSONB (PG-T5b.1.4)
var versionRangeJson = "{}";
if (doc.TryGetValue("versionRanges", out var rangesValue) && rangesValue is BsonArray)
{
versionRangeJson = rangesValue.ToJson();
}
string[]? versionsFixed = null;
if (doc.TryGetValue("versionRanges", out var rangesForFixed) && rangesForFixed is BsonArray rangesArr)
{
versionsFixed = rangesArr.OfType<BsonDocument>()
.Select(r => TryGetString(r, "fixedVersion"))
.Where(v => !string.IsNullOrEmpty(v))
.Select(v => v!)
.ToArray();
if (versionsFixed.Length == 0) versionsFixed = null;
}
result.Add(new AdvisoryAffectedEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
Ecosystem = ecosystem,
PackageName = identifier,
Purl = BuildPurl(ecosystem, identifier),
VersionRange = versionRangeJson,
VersionsAffected = null,
VersionsFixed = versionsFixed,
DatabaseSpecific = null,
CreatedAt = now
});
}
return result;
}
private static List<AdvisoryReferenceEntity> ConvertReferences(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
{
var result = new List<AdvisoryReferenceEntity>();
if (!payload.TryGetValue("references", out var referencesValue) || referencesValue is not BsonArray referencesArray)
{
return result;
}
foreach (var doc in referencesArray.OfType<BsonDocument>())
{
var url = doc.GetValue("url", defaultValue: null)?.AsString;
if (string.IsNullOrEmpty(url))
continue;
var kind = TryGetString(doc, "kind") ?? "web";
result.Add(new AdvisoryReferenceEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
RefType = kind,
Url = url,
CreatedAt = now
});
}
return result;
}
private static List<AdvisoryCreditEntity> ConvertCredits(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
{
var result = new List<AdvisoryCreditEntity>();
if (!payload.TryGetValue("credits", out var creditsValue) || creditsValue is not BsonArray creditsArray)
{
return result;
}
foreach (var doc in creditsArray.OfType<BsonDocument>())
{
var displayName = doc.GetValue("displayName", defaultValue: null)?.AsString;
if (string.IsNullOrEmpty(displayName))
continue;
var role = TryGetString(doc, "role");
string? contact = null;
if (doc.TryGetValue("contacts", out var contactsValue) && contactsValue is BsonArray contactsArray)
{
contact = contactsArray.OfType<BsonValue>()
.Where(v => v.IsString)
.Select(v => v.AsString)
.FirstOrDefault();
}
result.Add(new AdvisoryCreditEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
Name = displayName,
Contact = contact,
CreditType = role,
CreatedAt = now
});
}
return result;
}
private static List<AdvisoryWeaknessEntity> ConvertWeaknesses(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
{
var result = new List<AdvisoryWeaknessEntity>();
if (!payload.TryGetValue("cwes", out var cwesValue) || cwesValue is not BsonArray cwesArray)
{
return result;
}
foreach (var doc in cwesArray.OfType<BsonDocument>())
{
var identifier = doc.GetValue("identifier", defaultValue: null)?.AsString;
if (string.IsNullOrEmpty(identifier))
continue;
var name = TryGetString(doc, "name");
string? source = null;
if (doc.TryGetValue("provenance", out var provValue) && provValue.IsBsonDocument)
{
source = TryGetString(provValue.AsBsonDocument, "source");
}
result.Add(new AdvisoryWeaknessEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
CweId = identifier,
Description = name,
Source = source,
CreatedAt = now
});
}
return result;
}
private static List<KevFlagEntity> ConvertKevFlags(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
{
// KEV flags are typically stored separately; this handles inline KEV data if present
var result = new List<KevFlagEntity>();
// Check for exploitKnown flag
var exploitKnown = payload.TryGetValue("exploitKnown", out var exploitValue)
&& exploitValue.IsBoolean
&& exploitValue.AsBoolean;
if (!exploitKnown)
{
return result;
}
// Extract CVE ID for KEV flag
string? cveId = null;
if (payload.TryGetValue("aliases", out var aliasValue) && aliasValue is BsonArray aliasArray)
{
cveId = aliasArray.OfType<BsonValue>()
.Where(v => v.IsString && v.AsString.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
.Select(v => v.AsString)
.FirstOrDefault();
}
if (string.IsNullOrEmpty(cveId))
{
return result;
}
result.Add(new KevFlagEntity
{
Id = Guid.NewGuid(),
AdvisoryId = advisoryId,
CveId = cveId,
VendorProject = null,
Product = null,
VulnerabilityName = TryGetString(payload, "title"),
DateAdded = DateOnly.FromDateTime(now.UtcDateTime),
DueDate = null,
KnownRansomwareUse = false,
Notes = null,
CreatedAt = now
});
return result;
}
private static string MapTypeToEcosystem(string type)
{
return type.ToLowerInvariant() switch
{
"npm" => "npm",
"pypi" => "pypi",
"maven" => "maven",
"nuget" => "nuget",
"cargo" => "cargo",
"go" => "go",
"rubygems" => "rubygems",
"composer" => "composer",
"hex" => "hex",
"pub" => "pub",
"rpm" => "rpm",
"deb" => "deb",
"apk" => "apk",
"cpe" => "cpe",
"semver" => "generic",
"vendor" => "vendor",
"ics-vendor" => "ics",
_ => "generic"
};
}
private static string? BuildPurl(string ecosystem, string identifier)
{
// Only build PURL for supported ecosystems
return ecosystem switch
{
"npm" => $"pkg:npm/{identifier}",
"pypi" => $"pkg:pypi/{identifier}",
"maven" => identifier.Contains(':') ? $"pkg:maven/{identifier.Replace(':', '/')}" : null,
"nuget" => $"pkg:nuget/{identifier}",
"cargo" => $"pkg:cargo/{identifier}",
"go" => $"pkg:golang/{identifier}",
"rubygems" => $"pkg:gem/{identifier}",
"composer" => $"pkg:composer/{identifier}",
"hex" => $"pkg:hex/{identifier}",
"pub" => $"pkg:pub/{identifier}",
_ => null
};
}
private static string[]? ExtractFixedVersions(IEnumerable<AffectedVersionRange> ranges)
{
var fixedVersions = ranges
.Where(r => !string.IsNullOrEmpty(r.FixedVersion))
.Select(r => r.FixedVersion!)
.Distinct()
.ToArray();
return fixedVersions.Length > 0 ? fixedVersions : null;
}
private static string? TryGetString(BsonDocument doc, string field)
{
return doc.TryGetValue(field, out var value) && value.IsString ? value.AsString : null;
}
private static DateTimeOffset? TryReadDateTime(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value))
return null;
return value switch
{
BsonDateTime dateTime => DateTime.SpecifyKind(dateTime.ToUniversalTime(), DateTimeKind.Utc),
BsonString stringValue when DateTimeOffset.TryParse(stringValue.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null
};
}
}