feat(api): Implement Console Export Client and Models
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Findings Ledger CI / build-test (push) Has been cancelled
Findings Ledger CI / migration-validation (push) Has been cancelled
Findings Ledger CI / generate-manifest (push) Has been cancelled
mock-dev-release / package-mock-release (push) Has been cancelled
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Findings Ledger CI / build-test (push) Has been cancelled
Findings Ledger CI / migration-validation (push) Has been cancelled
Findings Ledger CI / generate-manifest (push) Has been cancelled
mock-dev-release / package-mock-release (push) Has been cancelled
- Added ConsoleExportClient for managing export requests and responses. - Introduced ConsoleExportRequest and ConsoleExportResponse models. - Implemented methods for creating and retrieving exports with appropriate headers. feat(crypto): Add Software SM2/SM3 Cryptography Provider - Implemented SmSoftCryptoProvider for software-only SM2/SM3 cryptography. - Added support for signing and verification using SM2 algorithm. - Included hashing functionality with SM3 algorithm. - Configured options for loading keys from files and environment gate checks. test(crypto): Add unit tests for SmSoftCryptoProvider - Created comprehensive tests for signing, verifying, and hashing functionalities. - Ensured correct behavior for key management and error handling. feat(api): Enhance Console Export Models - Expanded ConsoleExport models to include detailed status and event types. - Added support for various export formats and notification options. test(time): Implement TimeAnchorPolicyService tests - Developed tests for TimeAnchorPolicyService to validate time anchors. - Covered scenarios for anchor validation, drift calculation, and policy enforcement.
This commit is contained in:
@@ -1,21 +1,12 @@
|
||||
using System.Text.Json;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Storage.Postgres.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres.Conversion;
|
||||
|
||||
/// <summary>
|
||||
/// Converts MongoDB advisory documents to PostgreSQL entity structures.
|
||||
/// This converter handles the transformation from MongoDB's document-based storage
|
||||
/// to PostgreSQL's relational structure with normalized child tables.
|
||||
/// Converts domain advisories to PostgreSQL entity structures.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Task: PG-T5b.1.1 - Build AdvisoryConverter to parse MongoDB documents
|
||||
/// Task: PG-T5b.1.2 - Map to relational structure with child tables
|
||||
/// Task: PG-T5b.1.3 - Preserve provenance JSONB
|
||||
/// Task: PG-T5b.1.4 - Handle version ranges (keep as JSONB)
|
||||
/// </remarks>
|
||||
public sealed class AdvisoryConverter
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
@@ -25,86 +16,8 @@ public sealed class AdvisoryConverter
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Converts a MongoDB BsonDocument payload to PostgreSQL entities.
|
||||
/// Converts an Advisory domain model to PostgreSQL entities.
|
||||
/// </summary>
|
||||
/// <param name="payload">The MongoDB advisory payload (BsonDocument).</param>
|
||||
/// <param name="sourceId">Optional source ID to associate with the advisory.</param>
|
||||
/// <returns>A conversion result containing the main entity and all child entities.</returns>
|
||||
public AdvisoryConversionResult Convert(BsonDocument payload, Guid? sourceId = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(payload);
|
||||
|
||||
var advisoryKey = payload.GetValue("advisoryKey", defaultValue: null)?.AsString
|
||||
?? throw new InvalidOperationException("advisoryKey missing from payload.");
|
||||
|
||||
var title = payload.GetValue("title", defaultValue: null)?.AsString ?? advisoryKey;
|
||||
var summary = TryGetString(payload, "summary");
|
||||
var description = TryGetString(payload, "description");
|
||||
var severity = TryGetString(payload, "severity");
|
||||
var published = TryReadDateTime(payload, "published");
|
||||
var modified = TryReadDateTime(payload, "modified");
|
||||
|
||||
// Extract primary vulnerability ID from aliases (first CVE if available)
|
||||
var aliases = ExtractAliases(payload);
|
||||
var cveAlias = aliases.FirstOrDefault(a => a.AliasType == "cve");
|
||||
var firstAlias = aliases.FirstOrDefault();
|
||||
var primaryVulnId = cveAlias != default ? cveAlias.AliasValue
|
||||
: (firstAlias != default ? firstAlias.AliasValue : advisoryKey);
|
||||
|
||||
// Extract provenance and serialize to JSONB
|
||||
var provenanceJson = ExtractProvenanceJson(payload);
|
||||
|
||||
// Create the main advisory entity
|
||||
var advisoryId = Guid.NewGuid();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
var advisory = new AdvisoryEntity
|
||||
{
|
||||
Id = advisoryId,
|
||||
AdvisoryKey = advisoryKey,
|
||||
PrimaryVulnId = primaryVulnId,
|
||||
SourceId = sourceId,
|
||||
Title = title,
|
||||
Summary = summary,
|
||||
Description = description,
|
||||
Severity = severity,
|
||||
PublishedAt = published,
|
||||
ModifiedAt = modified,
|
||||
WithdrawnAt = null,
|
||||
Provenance = provenanceJson,
|
||||
RawPayload = payload.ToJson(),
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
// Convert all child entities
|
||||
var aliasEntities = ConvertAliases(advisoryId, aliases, now);
|
||||
var cvssEntities = ConvertCvss(advisoryId, payload, now);
|
||||
var affectedEntities = ConvertAffected(advisoryId, payload, now);
|
||||
var referenceEntities = ConvertReferences(advisoryId, payload, now);
|
||||
var creditEntities = ConvertCredits(advisoryId, payload, now);
|
||||
var weaknessEntities = ConvertWeaknesses(advisoryId, payload, now);
|
||||
var kevFlags = ConvertKevFlags(advisoryId, payload, now);
|
||||
|
||||
return new AdvisoryConversionResult
|
||||
{
|
||||
Advisory = advisory,
|
||||
Aliases = aliasEntities,
|
||||
Cvss = cvssEntities,
|
||||
Affected = affectedEntities,
|
||||
References = referenceEntities,
|
||||
Credits = creditEntities,
|
||||
Weaknesses = weaknessEntities,
|
||||
KevFlags = kevFlags
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts an Advisory domain model directly to PostgreSQL entities.
|
||||
/// </summary>
|
||||
/// <param name="advisory">The Advisory domain model.</param>
|
||||
/// <param name="sourceId">Optional source ID.</param>
|
||||
/// <returns>A conversion result containing all entities.</returns>
|
||||
public AdvisoryConversionResult ConvertFromDomain(Advisory advisory, Guid? sourceId = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(advisory);
|
||||
@@ -112,13 +25,11 @@ public sealed class AdvisoryConverter
|
||||
var advisoryId = Guid.NewGuid();
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Determine primary vulnerability ID
|
||||
var primaryVulnId = advisory.Aliases
|
||||
.FirstOrDefault(a => a.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
|
||||
?? advisory.Aliases.FirstOrDefault()
|
||||
?? advisory.AdvisoryKey;
|
||||
|
||||
// Serialize provenance to JSON
|
||||
var provenanceJson = JsonSerializer.Serialize(advisory.Provenance, JsonOptions);
|
||||
|
||||
var entity = new AdvisoryEntity
|
||||
@@ -140,7 +51,7 @@ public sealed class AdvisoryConverter
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
// Convert aliases
|
||||
// Aliases
|
||||
var aliasEntities = new List<AdvisoryAliasEntity>();
|
||||
var isPrimarySet = false;
|
||||
foreach (var alias in advisory.Aliases)
|
||||
@@ -160,7 +71,7 @@ public sealed class AdvisoryConverter
|
||||
});
|
||||
}
|
||||
|
||||
// Convert CVSS metrics
|
||||
// CVSS
|
||||
var cvssEntities = new List<AdvisoryCvssEntity>();
|
||||
var isPrimaryCvss = true;
|
||||
foreach (var metric in advisory.CvssMetrics)
|
||||
@@ -182,7 +93,7 @@ public sealed class AdvisoryConverter
|
||||
isPrimaryCvss = false;
|
||||
}
|
||||
|
||||
// Convert affected packages
|
||||
// Affected packages
|
||||
var affectedEntities = new List<AdvisoryAffectedEntity>();
|
||||
foreach (var pkg in advisory.AffectedPackages)
|
||||
{
|
||||
@@ -204,48 +115,60 @@ public sealed class AdvisoryConverter
|
||||
});
|
||||
}
|
||||
|
||||
// Convert references
|
||||
var referenceEntities = new List<AdvisoryReferenceEntity>();
|
||||
foreach (var reference in advisory.References)
|
||||
// References
|
||||
var referenceEntities = advisory.References.Select(reference => new AdvisoryReferenceEntity
|
||||
{
|
||||
referenceEntities.Add(new AdvisoryReferenceEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
RefType = reference.Kind ?? "web",
|
||||
Url = reference.Url,
|
||||
CreatedAt = now
|
||||
});
|
||||
}
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
RefType = reference.Kind ?? "web",
|
||||
Url = reference.Url,
|
||||
CreatedAt = now
|
||||
}).ToList();
|
||||
|
||||
// Convert credits
|
||||
var creditEntities = new List<AdvisoryCreditEntity>();
|
||||
foreach (var credit in advisory.Credits)
|
||||
// Credits
|
||||
var creditEntities = advisory.Credits.Select(credit => new AdvisoryCreditEntity
|
||||
{
|
||||
creditEntities.Add(new AdvisoryCreditEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
Name = credit.DisplayName,
|
||||
Contact = credit.Contacts.FirstOrDefault(),
|
||||
CreditType = credit.Role,
|
||||
CreatedAt = now
|
||||
});
|
||||
}
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
Name = credit.DisplayName,
|
||||
Contact = credit.Contacts.FirstOrDefault(),
|
||||
CreditType = credit.Role,
|
||||
CreatedAt = now
|
||||
}).ToList();
|
||||
|
||||
// Convert weaknesses
|
||||
var weaknessEntities = new List<AdvisoryWeaknessEntity>();
|
||||
foreach (var weakness in advisory.Cwes)
|
||||
// Weaknesses
|
||||
var weaknessEntities = advisory.Cwes.Select(weakness => new AdvisoryWeaknessEntity
|
||||
{
|
||||
weaknessEntities.Add(new AdvisoryWeaknessEntity
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CweId = weakness.Identifier,
|
||||
Description = weakness.Name,
|
||||
Source = weakness.Provenance.FirstOrDefault()?.Source,
|
||||
CreatedAt = now
|
||||
}).ToList();
|
||||
|
||||
// KEV flags from domain data
|
||||
var kevFlags = new List<KevFlagEntity>();
|
||||
if (advisory.ExploitKnown)
|
||||
{
|
||||
var cveId = advisory.Aliases.FirstOrDefault(a => a.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase));
|
||||
if (!string.IsNullOrWhiteSpace(cveId))
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CweId = weakness.Identifier,
|
||||
Description = weakness.Name,
|
||||
Source = weakness.Provenance.FirstOrDefault()?.Source,
|
||||
CreatedAt = now
|
||||
});
|
||||
kevFlags.Add(new KevFlagEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CveId = cveId,
|
||||
VendorProject = null,
|
||||
Product = null,
|
||||
VulnerabilityName = advisory.Title,
|
||||
DateAdded = DateOnly.FromDateTime(now.UtcDateTime),
|
||||
DueDate = null,
|
||||
KnownRansomwareUse = false,
|
||||
Notes = null,
|
||||
CreatedAt = now
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return new AdvisoryConversionResult
|
||||
@@ -257,32 +180,10 @@ public sealed class AdvisoryConverter
|
||||
References = referenceEntities,
|
||||
Credits = creditEntities,
|
||||
Weaknesses = weaknessEntities,
|
||||
KevFlags = new List<KevFlagEntity>()
|
||||
KevFlags = kevFlags
|
||||
};
|
||||
}
|
||||
|
||||
private static List<(string AliasType, string AliasValue, bool IsPrimary)> ExtractAliases(BsonDocument payload)
|
||||
{
|
||||
var result = new List<(string AliasType, string AliasValue, bool IsPrimary)>();
|
||||
|
||||
if (!payload.TryGetValue("aliases", out var aliasValue) || aliasValue is not BsonArray aliasArray)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
var isPrimarySet = false;
|
||||
foreach (var alias in aliasArray.OfType<BsonValue>().Where(x => x.IsString).Select(x => x.AsString))
|
||||
{
|
||||
var aliasType = DetermineAliasType(alias);
|
||||
var isPrimary = !isPrimarySet && aliasType == "cve";
|
||||
if (isPrimary) isPrimarySet = true;
|
||||
|
||||
result.Add((aliasType, alias, isPrimary));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string DetermineAliasType(string alias)
|
||||
{
|
||||
if (alias.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -305,288 +206,8 @@ public sealed class AdvisoryConverter
|
||||
return "other";
|
||||
}
|
||||
|
||||
private static string ExtractProvenanceJson(BsonDocument payload)
|
||||
{
|
||||
if (!payload.TryGetValue("provenance", out var provenanceValue) || provenanceValue is not BsonArray provenanceArray)
|
||||
{
|
||||
return "[]";
|
||||
}
|
||||
|
||||
return provenanceArray.ToJson();
|
||||
}
|
||||
|
||||
private static List<AdvisoryAliasEntity> ConvertAliases(
|
||||
Guid advisoryId,
|
||||
List<(string AliasType, string AliasValue, bool IsPrimary)> aliases,
|
||||
DateTimeOffset now)
|
||||
{
|
||||
return aliases.Select(a => new AdvisoryAliasEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
AliasType = a.AliasType,
|
||||
AliasValue = a.AliasValue,
|
||||
IsPrimary = a.IsPrimary,
|
||||
CreatedAt = now
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
private static List<AdvisoryCvssEntity> ConvertCvss(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
|
||||
{
|
||||
var result = new List<AdvisoryCvssEntity>();
|
||||
|
||||
if (!payload.TryGetValue("cvssMetrics", out var cvssValue) || cvssValue is not BsonArray cvssArray)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
var isPrimary = true;
|
||||
foreach (var doc in cvssArray.OfType<BsonDocument>())
|
||||
{
|
||||
var version = doc.GetValue("version", defaultValue: null)?.AsString;
|
||||
var vector = doc.GetValue("vector", defaultValue: null)?.AsString;
|
||||
var baseScore = doc.TryGetValue("baseScore", out var scoreValue) && scoreValue.IsNumeric
|
||||
? (decimal)scoreValue.ToDouble()
|
||||
: 0m;
|
||||
var baseSeverity = TryGetString(doc, "baseSeverity");
|
||||
var source = doc.TryGetValue("provenance", out var provValue) && provValue.IsBsonDocument
|
||||
? TryGetString(provValue.AsBsonDocument, "source")
|
||||
: null;
|
||||
|
||||
if (string.IsNullOrEmpty(version) || string.IsNullOrEmpty(vector))
|
||||
continue;
|
||||
|
||||
result.Add(new AdvisoryCvssEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CvssVersion = version,
|
||||
VectorString = vector,
|
||||
BaseScore = baseScore,
|
||||
BaseSeverity = baseSeverity,
|
||||
ExploitabilityScore = null,
|
||||
ImpactScore = null,
|
||||
Source = source,
|
||||
IsPrimary = isPrimary,
|
||||
CreatedAt = now
|
||||
});
|
||||
isPrimary = false;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<AdvisoryAffectedEntity> ConvertAffected(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
|
||||
{
|
||||
var result = new List<AdvisoryAffectedEntity>();
|
||||
|
||||
if (!payload.TryGetValue("affectedPackages", out var affectedValue) || affectedValue is not BsonArray affectedArray)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
foreach (var doc in affectedArray.OfType<BsonDocument>())
|
||||
{
|
||||
var type = doc.GetValue("type", defaultValue: null)?.AsString ?? "semver";
|
||||
var identifier = doc.GetValue("identifier", defaultValue: null)?.AsString;
|
||||
|
||||
if (string.IsNullOrEmpty(identifier))
|
||||
continue;
|
||||
|
||||
var ecosystem = MapTypeToEcosystem(type);
|
||||
|
||||
// Version ranges kept as JSONB (PG-T5b.1.4)
|
||||
var versionRangeJson = "{}";
|
||||
if (doc.TryGetValue("versionRanges", out var rangesValue) && rangesValue is BsonArray)
|
||||
{
|
||||
versionRangeJson = rangesValue.ToJson();
|
||||
}
|
||||
|
||||
string[]? versionsFixed = null;
|
||||
if (doc.TryGetValue("versionRanges", out var rangesForFixed) && rangesForFixed is BsonArray rangesArr)
|
||||
{
|
||||
versionsFixed = rangesArr.OfType<BsonDocument>()
|
||||
.Select(r => TryGetString(r, "fixedVersion"))
|
||||
.Where(v => !string.IsNullOrEmpty(v))
|
||||
.Select(v => v!)
|
||||
.ToArray();
|
||||
if (versionsFixed.Length == 0) versionsFixed = null;
|
||||
}
|
||||
|
||||
result.Add(new AdvisoryAffectedEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
Ecosystem = ecosystem,
|
||||
PackageName = identifier,
|
||||
Purl = BuildPurl(ecosystem, identifier),
|
||||
VersionRange = versionRangeJson,
|
||||
VersionsAffected = null,
|
||||
VersionsFixed = versionsFixed,
|
||||
DatabaseSpecific = null,
|
||||
CreatedAt = now
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<AdvisoryReferenceEntity> ConvertReferences(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
|
||||
{
|
||||
var result = new List<AdvisoryReferenceEntity>();
|
||||
|
||||
if (!payload.TryGetValue("references", out var referencesValue) || referencesValue is not BsonArray referencesArray)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
foreach (var doc in referencesArray.OfType<BsonDocument>())
|
||||
{
|
||||
var url = doc.GetValue("url", defaultValue: null)?.AsString;
|
||||
if (string.IsNullOrEmpty(url))
|
||||
continue;
|
||||
|
||||
var kind = TryGetString(doc, "kind") ?? "web";
|
||||
|
||||
result.Add(new AdvisoryReferenceEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
RefType = kind,
|
||||
Url = url,
|
||||
CreatedAt = now
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<AdvisoryCreditEntity> ConvertCredits(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
|
||||
{
|
||||
var result = new List<AdvisoryCreditEntity>();
|
||||
|
||||
if (!payload.TryGetValue("credits", out var creditsValue) || creditsValue is not BsonArray creditsArray)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
foreach (var doc in creditsArray.OfType<BsonDocument>())
|
||||
{
|
||||
var displayName = doc.GetValue("displayName", defaultValue: null)?.AsString;
|
||||
if (string.IsNullOrEmpty(displayName))
|
||||
continue;
|
||||
|
||||
var role = TryGetString(doc, "role");
|
||||
string? contact = null;
|
||||
if (doc.TryGetValue("contacts", out var contactsValue) && contactsValue is BsonArray contactsArray)
|
||||
{
|
||||
contact = contactsArray.OfType<BsonValue>()
|
||||
.Where(v => v.IsString)
|
||||
.Select(v => v.AsString)
|
||||
.FirstOrDefault();
|
||||
}
|
||||
|
||||
result.Add(new AdvisoryCreditEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
Name = displayName,
|
||||
Contact = contact,
|
||||
CreditType = role,
|
||||
CreatedAt = now
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<AdvisoryWeaknessEntity> ConvertWeaknesses(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
|
||||
{
|
||||
var result = new List<AdvisoryWeaknessEntity>();
|
||||
|
||||
if (!payload.TryGetValue("cwes", out var cwesValue) || cwesValue is not BsonArray cwesArray)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
foreach (var doc in cwesArray.OfType<BsonDocument>())
|
||||
{
|
||||
var identifier = doc.GetValue("identifier", defaultValue: null)?.AsString;
|
||||
if (string.IsNullOrEmpty(identifier))
|
||||
continue;
|
||||
|
||||
var name = TryGetString(doc, "name");
|
||||
string? source = null;
|
||||
if (doc.TryGetValue("provenance", out var provValue) && provValue.IsBsonDocument)
|
||||
{
|
||||
source = TryGetString(provValue.AsBsonDocument, "source");
|
||||
}
|
||||
|
||||
result.Add(new AdvisoryWeaknessEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CweId = identifier,
|
||||
Description = name,
|
||||
Source = source,
|
||||
CreatedAt = now
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<KevFlagEntity> ConvertKevFlags(Guid advisoryId, BsonDocument payload, DateTimeOffset now)
|
||||
{
|
||||
// KEV flags are typically stored separately; this handles inline KEV data if present
|
||||
var result = new List<KevFlagEntity>();
|
||||
|
||||
// Check for exploitKnown flag
|
||||
var exploitKnown = payload.TryGetValue("exploitKnown", out var exploitValue)
|
||||
&& exploitValue.IsBoolean
|
||||
&& exploitValue.AsBoolean;
|
||||
|
||||
if (!exploitKnown)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
// Extract CVE ID for KEV flag
|
||||
string? cveId = null;
|
||||
if (payload.TryGetValue("aliases", out var aliasValue) && aliasValue is BsonArray aliasArray)
|
||||
{
|
||||
cveId = aliasArray.OfType<BsonValue>()
|
||||
.Where(v => v.IsString && v.AsString.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase))
|
||||
.Select(v => v.AsString)
|
||||
.FirstOrDefault();
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(cveId))
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
result.Add(new KevFlagEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CveId = cveId,
|
||||
VendorProject = null,
|
||||
Product = null,
|
||||
VulnerabilityName = TryGetString(payload, "title"),
|
||||
DateAdded = DateOnly.FromDateTime(now.UtcDateTime),
|
||||
DueDate = null,
|
||||
KnownRansomwareUse = false,
|
||||
Notes = null,
|
||||
CreatedAt = now
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string MapTypeToEcosystem(string type)
|
||||
{
|
||||
return type.ToLowerInvariant() switch
|
||||
private static string MapTypeToEcosystem(string type) =>
|
||||
type.ToLowerInvariant() switch
|
||||
{
|
||||
"npm" => "npm",
|
||||
"pypi" => "pypi",
|
||||
@@ -607,12 +228,9 @@ public sealed class AdvisoryConverter
|
||||
"ics-vendor" => "ics",
|
||||
_ => "generic"
|
||||
};
|
||||
}
|
||||
|
||||
private static string? BuildPurl(string ecosystem, string identifier)
|
||||
{
|
||||
// Only build PURL for supported ecosystems
|
||||
return ecosystem switch
|
||||
private static string? BuildPurl(string ecosystem, string identifier) =>
|
||||
ecosystem switch
|
||||
{
|
||||
"npm" => $"pkg:npm/{identifier}",
|
||||
"pypi" => $"pkg:pypi/{identifier}",
|
||||
@@ -626,7 +244,6 @@ public sealed class AdvisoryConverter
|
||||
"pub" => $"pkg:pub/{identifier}",
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private static string[]? ExtractFixedVersions(IEnumerable<AffectedVersionRange> ranges)
|
||||
{
|
||||
@@ -638,22 +255,4 @@ public sealed class AdvisoryConverter
|
||||
|
||||
return fixedVersions.Length > 0 ? fixedVersions : null;
|
||||
}
|
||||
|
||||
private static string? TryGetString(BsonDocument doc, string field)
|
||||
{
|
||||
return doc.TryGetValue(field, out var value) && value.IsString ? value.AsString : null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? TryReadDateTime(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value))
|
||||
return null;
|
||||
|
||||
return value switch
|
||||
{
|
||||
BsonDateTime dateTime => DateTime.SpecifyKind(dateTime.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonString stringValue when DateTimeOffset.TryParse(stringValue.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Postgres.Models;
|
||||
using StellaOps.Concelier.Storage.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres.Converters;
|
||||
|
||||
/// <summary>
|
||||
/// Service to convert Mongo advisory documents and persist them into PostgreSQL.
|
||||
/// </summary>
|
||||
public sealed class AdvisoryConversionService
|
||||
{
|
||||
private readonly IAdvisoryRepository _advisories;
|
||||
|
||||
public AdvisoryConversionService(IAdvisoryRepository advisories)
|
||||
{
|
||||
_advisories = advisories;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts a Mongo advisory document and persists it (upsert) with all child rows.
|
||||
/// </summary>
|
||||
public Task<AdvisoryEntity> ConvertAndUpsertAsync(
|
||||
AdvisoryDocument doc,
|
||||
string sourceKey,
|
||||
Guid sourceId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var result = AdvisoryConverter.Convert(doc, sourceKey, sourceId);
|
||||
return _advisories.UpsertAsync(
|
||||
result.Advisory,
|
||||
result.Aliases,
|
||||
result.Cvss,
|
||||
result.Affected,
|
||||
result.References,
|
||||
result.Credits,
|
||||
result.Weaknesses,
|
||||
result.KevFlags,
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -1,297 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Postgres.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres.Converters;
|
||||
|
||||
/// <summary>
|
||||
/// Converts Mongo advisory documents to Postgres advisory entities and child collections.
|
||||
/// Deterministic: ordering of child collections is preserved (sorted for stable SQL writes).
|
||||
/// </summary>
|
||||
public static class AdvisoryConverter
|
||||
{
|
||||
public sealed record Result(
|
||||
AdvisoryEntity Advisory,
|
||||
IReadOnlyList<AdvisoryAliasEntity> Aliases,
|
||||
IReadOnlyList<AdvisoryCvssEntity> Cvss,
|
||||
IReadOnlyList<AdvisoryAffectedEntity> Affected,
|
||||
IReadOnlyList<AdvisoryReferenceEntity> References,
|
||||
IReadOnlyList<AdvisoryCreditEntity> Credits,
|
||||
IReadOnlyList<AdvisoryWeaknessEntity> Weaknesses,
|
||||
IReadOnlyList<KevFlagEntity> KevFlags);
|
||||
|
||||
/// <summary>
|
||||
/// Maps a Mongo AdvisoryDocument and its raw payload into Postgres entities.
|
||||
/// </summary>
|
||||
public static Result Convert(
|
||||
AdvisoryDocument doc,
|
||||
string sourceKey,
|
||||
Guid sourceId,
|
||||
string? contentHash = null)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
// Top-level advisory
|
||||
var advisoryId = Guid.NewGuid();
|
||||
var payloadJson = doc.Payload.ToJson();
|
||||
var provenanceJson = JsonSerializer.Serialize(new { source = sourceKey });
|
||||
|
||||
var advisory = new AdvisoryEntity
|
||||
{
|
||||
Id = advisoryId,
|
||||
AdvisoryKey = doc.AdvisoryKey,
|
||||
PrimaryVulnId = doc.Payload.GetValue("primaryVulnId", doc.AdvisoryKey)?.ToString() ?? doc.AdvisoryKey,
|
||||
SourceId = sourceId,
|
||||
Title = doc.Payload.GetValue("title", null)?.ToString(),
|
||||
Summary = doc.Payload.GetValue("summary", null)?.ToString(),
|
||||
Description = doc.Payload.GetValue("description", null)?.ToString(),
|
||||
Severity = doc.Payload.GetValue("severity", null)?.ToString(),
|
||||
PublishedAt = doc.Published.HasValue ? DateTime.SpecifyKind(doc.Published.Value, DateTimeKind.Utc) : null,
|
||||
ModifiedAt = DateTime.SpecifyKind(doc.Modified, DateTimeKind.Utc),
|
||||
WithdrawnAt = doc.Payload.TryGetValue("withdrawnAt", out var withdrawn) && withdrawn.IsValidDateTime
|
||||
? withdrawn.ToUniversalTime()
|
||||
: null,
|
||||
Provenance = provenanceJson,
|
||||
RawPayload = payloadJson,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now
|
||||
};
|
||||
|
||||
// Aliases
|
||||
var aliases = doc.Payload.TryGetValue("aliases", out var aliasesBson) && aliasesBson.IsBsonArray
|
||||
? aliasesBson.AsBsonArray.Select(v => v.ToString() ?? string.Empty)
|
||||
: Enumerable.Empty<string>();
|
||||
|
||||
var aliasEntities = aliases
|
||||
.Where(a => !string.IsNullOrWhiteSpace(a))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(a => a, StringComparer.OrdinalIgnoreCase)
|
||||
.Select((alias, idx) => new AdvisoryAliasEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
AliasType = alias.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase) ? "CVE" : "OTHER",
|
||||
AliasValue = alias,
|
||||
IsPrimary = idx == 0,
|
||||
CreatedAt = now
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
// CVSS
|
||||
var cvssEntities = BuildCvssEntities(doc, advisoryId, now);
|
||||
|
||||
// Affected
|
||||
var affectedEntities = BuildAffectedEntities(doc, advisoryId, now);
|
||||
|
||||
// References
|
||||
var referencesEntities = BuildReferenceEntities(doc, advisoryId, now);
|
||||
|
||||
// Credits
|
||||
var creditEntities = BuildCreditEntities(doc, advisoryId, now);
|
||||
|
||||
// Weaknesses
|
||||
var weaknessEntities = BuildWeaknessEntities(doc, advisoryId, now);
|
||||
|
||||
// KEV flags (from payload.kev if present)
|
||||
var kevEntities = BuildKevEntities(doc, advisoryId, now);
|
||||
|
||||
return new Result(
|
||||
advisory,
|
||||
aliasEntities,
|
||||
cvssEntities,
|
||||
affectedEntities,
|
||||
referencesEntities,
|
||||
creditEntities,
|
||||
weaknessEntities,
|
||||
kevEntities);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryCvssEntity> BuildCvssEntities(AdvisoryDocument doc, Guid advisoryId, DateTimeOffset now)
|
||||
{
|
||||
if (!doc.Payload.TryGetValue("cvss", out var cvssValue) || !cvssValue.IsBsonArray)
|
||||
{
|
||||
return Array.Empty<AdvisoryCvssEntity>();
|
||||
}
|
||||
|
||||
return cvssValue.AsBsonArray
|
||||
.Where(v => v.IsBsonDocument)
|
||||
.Select(v => v.AsBsonDocument)
|
||||
.Select(d => new AdvisoryCvssEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CvssVersion = d.GetValue("version", "3.1").ToString() ?? "3.1",
|
||||
VectorString = d.GetValue("vector", string.Empty).ToString() ?? string.Empty,
|
||||
BaseScore = d.GetValue("baseScore", 0m).ToDecimal(),
|
||||
BaseSeverity = d.GetValue("baseSeverity", null)?.ToString(),
|
||||
ExploitabilityScore = d.GetValue("exploitabilityScore", null)?.ToNullableDecimal(),
|
||||
ImpactScore = d.GetValue("impactScore", null)?.ToNullableDecimal(),
|
||||
Source = d.GetValue("source", null)?.ToString(),
|
||||
IsPrimary = d.GetValue("isPrimary", false).ToBoolean(),
|
||||
CreatedAt = now
|
||||
})
|
||||
.OrderByDescending(c => c.IsPrimary)
|
||||
.ThenByDescending(c => c.BaseScore)
|
||||
.ThenBy(c => c.Id)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryAffectedEntity> BuildAffectedEntities(AdvisoryDocument doc, Guid advisoryId, DateTimeOffset now)
|
||||
{
|
||||
if (!doc.Payload.TryGetValue("affected", out var affectedValue) || !affectedValue.IsBsonArray)
|
||||
{
|
||||
return Array.Empty<AdvisoryAffectedEntity>();
|
||||
}
|
||||
|
||||
return affectedValue.AsBsonArray
|
||||
.Where(v => v.IsBsonDocument)
|
||||
.Select(v => v.AsBsonDocument)
|
||||
.Select(d => new AdvisoryAffectedEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
Ecosystem = d.GetValue("ecosystem", string.Empty).ToString() ?? string.Empty,
|
||||
PackageName = d.GetValue("packageName", string.Empty).ToString() ?? string.Empty,
|
||||
Purl = d.GetValue("purl", null)?.ToString(),
|
||||
VersionRange = d.GetValue("range", "{}").ToString() ?? "{}",
|
||||
VersionsAffected = d.TryGetValue("versionsAffected", out var va) && va.IsBsonArray
|
||||
? va.AsBsonArray.Select(x => x.ToString() ?? string.Empty).ToArray()
|
||||
: null,
|
||||
VersionsFixed = d.TryGetValue("versionsFixed", out var vf) && vf.IsBsonArray
|
||||
? vf.AsBsonArray.Select(x => x.ToString() ?? string.Empty).ToArray()
|
||||
: null,
|
||||
DatabaseSpecific = d.GetValue("databaseSpecific", null)?.ToString(),
|
||||
CreatedAt = now
|
||||
})
|
||||
.OrderBy(a => a.Ecosystem)
|
||||
.ThenBy(a => a.PackageName)
|
||||
.ThenBy(a => a.Purl)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReferenceEntity> BuildReferenceEntities(AdvisoryDocument doc, Guid advisoryId, DateTimeOffset now)
|
||||
{
|
||||
if (!doc.Payload.TryGetValue("references", out var referencesValue) || !referencesValue.IsBsonArray)
|
||||
{
|
||||
return Array.Empty<AdvisoryReferenceEntity>();
|
||||
}
|
||||
|
||||
return referencesValue.AsBsonArray
|
||||
.Where(v => v.IsBsonDocument)
|
||||
.Select(v => v.AsBsonDocument)
|
||||
.Select(r => new AdvisoryReferenceEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
RefType = r.GetValue("type", "advisory").ToString() ?? "advisory",
|
||||
Url = r.GetValue("url", string.Empty).ToString() ?? string.Empty,
|
||||
CreatedAt = now
|
||||
})
|
||||
.OrderBy(r => r.Url)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryCreditEntity> BuildCreditEntities(AdvisoryDocument doc, Guid advisoryId, DateTimeOffset now)
|
||||
{
|
||||
if (!doc.Payload.TryGetValue("credits", out var creditsValue) || !creditsValue.IsBsonArray)
|
||||
{
|
||||
return Array.Empty<AdvisoryCreditEntity>();
|
||||
}
|
||||
|
||||
return creditsValue.AsBsonArray
|
||||
.Where(v => v.IsBsonDocument)
|
||||
.Select(v => v.AsBsonDocument)
|
||||
.Select(c => new AdvisoryCreditEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
Name = c.GetValue("name", string.Empty).ToString() ?? string.Empty,
|
||||
Contact = c.GetValue("contact", null)?.ToString(),
|
||||
CreditType = c.GetValue("type", null)?.ToString(),
|
||||
CreatedAt = now
|
||||
})
|
||||
.OrderBy(c => c.Name)
|
||||
.ThenBy(c => c.Contact)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryWeaknessEntity> BuildWeaknessEntities(AdvisoryDocument doc, Guid advisoryId, DateTimeOffset now)
|
||||
{
|
||||
if (!doc.Payload.TryGetValue("weaknesses", out var weaknessesValue) || !weaknessesValue.IsBsonArray)
|
||||
{
|
||||
return Array.Empty<AdvisoryWeaknessEntity>();
|
||||
}
|
||||
|
||||
return weaknessesValue.AsBsonArray
|
||||
.Where(v => v.IsBsonDocument)
|
||||
.Select(v => v.AsBsonDocument)
|
||||
.Select(w => new AdvisoryWeaknessEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CweId = w.GetValue("cweId", string.Empty).ToString() ?? string.Empty,
|
||||
Description = w.GetValue("description", null)?.ToString(),
|
||||
Source = w.GetValue("source", null)?.ToString(),
|
||||
CreatedAt = now
|
||||
})
|
||||
.OrderBy(w => w.CweId)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<KevFlagEntity> BuildKevEntities(AdvisoryDocument doc, Guid advisoryId, DateTimeOffset now)
|
||||
{
|
||||
if (!doc.Payload.TryGetValue("kev", out var kevValue) || !kevValue.IsBsonArray)
|
||||
{
|
||||
return Array.Empty<KevFlagEntity>();
|
||||
}
|
||||
|
||||
var today = DateOnly.FromDateTime(now.UtcDateTime);
|
||||
return kevValue.AsBsonArray
|
||||
.Where(v => v.IsBsonDocument)
|
||||
.Select(v => v.AsBsonDocument)
|
||||
.Select(k => new KevFlagEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
AdvisoryId = advisoryId,
|
||||
CveId = k.GetValue("cveId", string.Empty).ToString() ?? string.Empty,
|
||||
VendorProject = k.GetValue("vendorProject", null)?.ToString(),
|
||||
Product = k.GetValue("product", null)?.ToString(),
|
||||
VulnerabilityName = k.GetValue("name", null)?.ToString(),
|
||||
DateAdded = k.TryGetValue("dateAdded", out var dateAdded) && dateAdded.IsValidDateTime
|
||||
? DateOnly.FromDateTime(dateAdded.ToUniversalTime().Date)
|
||||
: today,
|
||||
DueDate = k.TryGetValue("dueDate", out var dueDate) && dueDate.IsValidDateTime
|
||||
? DateOnly.FromDateTime(dueDate.ToUniversalTime().Date)
|
||||
: null,
|
||||
KnownRansomwareUse = k.GetValue("knownRansomwareUse", false).ToBoolean(),
|
||||
Notes = k.GetValue("notes", null)?.ToString(),
|
||||
CreatedAt = now
|
||||
})
|
||||
.OrderBy(k => k.CveId)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static decimal ToDecimal(this object value)
|
||||
=> value switch
|
||||
{
|
||||
decimal d => d,
|
||||
double d => (decimal)d,
|
||||
float f => (decimal)f,
|
||||
IConvertible c => c.ToDecimal(null),
|
||||
_ => 0m
|
||||
};
|
||||
|
||||
private static decimal? ToNullableDecimal(this object? value)
|
||||
{
|
||||
if (value is null) return null;
|
||||
return value switch
|
||||
{
|
||||
decimal d => d,
|
||||
double d => (decimal)d,
|
||||
float f => (decimal)f,
|
||||
IConvertible c => c.ToDecimal(null),
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Postgres.Models;
|
||||
using StellaOps.Concelier.Storage.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres.Converters.Importers;
|
||||
|
||||
/// <summary>
|
||||
/// Imports GHSA/vendor advisories from Mongo into PostgreSQL.
|
||||
/// </summary>
|
||||
public sealed class GhsaImporter
|
||||
{
|
||||
private readonly IMongoCollection<AdvisoryDocument> _collection;
|
||||
private readonly AdvisoryConversionService _conversionService;
|
||||
private readonly IFeedSnapshotRepository _feedSnapshots;
|
||||
private readonly IAdvisorySnapshotRepository _advisorySnapshots;
|
||||
|
||||
public GhsaImporter(
|
||||
IMongoCollection<AdvisoryDocument> collection,
|
||||
AdvisoryConversionService conversionService,
|
||||
IFeedSnapshotRepository feedSnapshots,
|
||||
IAdvisorySnapshotRepository advisorySnapshots)
|
||||
{
|
||||
_collection = collection;
|
||||
_conversionService = conversionService;
|
||||
_feedSnapshots = feedSnapshots;
|
||||
_advisorySnapshots = advisorySnapshots;
|
||||
}
|
||||
|
||||
public async Task ImportSnapshotAsync(
|
||||
Guid sourceId,
|
||||
string sourceKey,
|
||||
string snapshotId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var advisories = await _collection
|
||||
.Find(Builders<AdvisoryDocument>.Filter.Empty)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var feedSnapshot = await _feedSnapshots.InsertAsync(new FeedSnapshotEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
SourceId = sourceId,
|
||||
SnapshotId = snapshotId,
|
||||
AdvisoryCount = advisories.Count,
|
||||
Metadata = $"{{\"source\":\"{sourceKey}\"}}",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var advisory in advisories)
|
||||
{
|
||||
var stored = await _conversionService.ConvertAndUpsertAsync(advisory, sourceKey, sourceId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _advisorySnapshots.InsertAsync(new AdvisorySnapshotEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
FeedSnapshotId = feedSnapshot.Id,
|
||||
AdvisoryKey = stored.AdvisoryKey,
|
||||
ContentHash = advisory.Payload.GetValue("hash", advisory.AdvisoryKey)?.ToString() ?? advisory.AdvisoryKey,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
using System.Text.Json;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Postgres.Models;
|
||||
using StellaOps.Concelier.Storage.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres.Converters.Importers;
|
||||
|
||||
/// <summary>
|
||||
/// Imports NVD advisory documents from Mongo into PostgreSQL using the advisory converter.
|
||||
/// </summary>
|
||||
public sealed class NvdImporter
|
||||
{
|
||||
private readonly IMongoCollection<AdvisoryDocument> _collection;
|
||||
private readonly AdvisoryConversionService _conversionService;
|
||||
private readonly IFeedSnapshotRepository _feedSnapshots;
|
||||
private readonly IAdvisorySnapshotRepository _advisorySnapshots;
|
||||
|
||||
public NvdImporter(
|
||||
IMongoCollection<AdvisoryDocument> collection,
|
||||
AdvisoryConversionService conversionService,
|
||||
IFeedSnapshotRepository feedSnapshots,
|
||||
IAdvisorySnapshotRepository advisorySnapshots)
|
||||
{
|
||||
_collection = collection;
|
||||
_conversionService = conversionService;
|
||||
_feedSnapshots = feedSnapshots;
|
||||
_advisorySnapshots = advisorySnapshots;
|
||||
}
|
||||
|
||||
public async Task ImportSnapshotAsync(
|
||||
Guid sourceId,
|
||||
string sourceKey,
|
||||
string snapshotId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var advisories = await _collection
|
||||
.Find(Builders<AdvisoryDocument>.Filter.Empty)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var feedSnapshot = await _feedSnapshots.InsertAsync(new FeedSnapshotEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
SourceId = sourceId,
|
||||
SnapshotId = snapshotId,
|
||||
AdvisoryCount = advisories.Count,
|
||||
Checksum = null,
|
||||
Metadata = JsonSerializer.Serialize(new { source = sourceKey, snapshot = snapshotId }),
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var advisory in advisories)
|
||||
{
|
||||
var stored = await _conversionService.ConvertAndUpsertAsync(advisory, sourceKey, sourceId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _advisorySnapshots.InsertAsync(new AdvisorySnapshotEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
FeedSnapshotId = feedSnapshot.Id,
|
||||
AdvisoryKey = stored.AdvisoryKey,
|
||||
ContentHash = advisory.Payload.GetValue("hash", advisory.AdvisoryKey)?.ToString() ?? advisory.AdvisoryKey,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Storage.Mongo.Advisories;
|
||||
using StellaOps.Concelier.Storage.Postgres.Models;
|
||||
using StellaOps.Concelier.Storage.Postgres.Repositories;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Postgres.Converters.Importers;
|
||||
|
||||
/// <summary>
|
||||
/// Imports OSV advisories from Mongo into PostgreSQL.
|
||||
/// </summary>
|
||||
public sealed class OsvImporter
|
||||
{
|
||||
private readonly IMongoCollection<AdvisoryDocument> _collection;
|
||||
private readonly AdvisoryConversionService _conversionService;
|
||||
private readonly IFeedSnapshotRepository _feedSnapshots;
|
||||
private readonly IAdvisorySnapshotRepository _advisorySnapshots;
|
||||
|
||||
public OsvImporter(
|
||||
IMongoCollection<AdvisoryDocument> collection,
|
||||
AdvisoryConversionService conversionService,
|
||||
IFeedSnapshotRepository feedSnapshots,
|
||||
IAdvisorySnapshotRepository advisorySnapshots)
|
||||
{
|
||||
_collection = collection;
|
||||
_conversionService = conversionService;
|
||||
_feedSnapshots = feedSnapshots;
|
||||
_advisorySnapshots = advisorySnapshots;
|
||||
}
|
||||
|
||||
public async Task ImportSnapshotAsync(
|
||||
Guid sourceId,
|
||||
string snapshotId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var advisories = await _collection
|
||||
.Find(Builders<AdvisoryDocument>.Filter.Empty)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var feedSnapshot = await _feedSnapshots.InsertAsync(new FeedSnapshotEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
SourceId = sourceId,
|
||||
SnapshotId = snapshotId,
|
||||
AdvisoryCount = advisories.Count,
|
||||
Metadata = "{\"source\":\"osv\"}",
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var advisory in advisories)
|
||||
{
|
||||
var stored = await _conversionService.ConvertAndUpsertAsync(advisory, "osv", sourceId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _advisorySnapshots.InsertAsync(new AdvisorySnapshotEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
FeedSnapshotId = feedSnapshot.Id,
|
||||
AdvisoryKey = stored.AdvisoryKey,
|
||||
ContentHash = advisory.Payload.GetValue("hash", advisory.AdvisoryKey)?.ToString() ?? advisory.AdvisoryKey,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,9 +15,11 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Exclude legacy Mongo-based import/conversion helpers until Postgres-native pipeline is ready -->
|
||||
<!-- Exclude legacy Mongo importers/converters; domain-based converter remains -->
|
||||
<Compile Remove="Converters\**\*.cs" />
|
||||
<Compile Remove="Conversion\**\*.cs" />
|
||||
<Compile Remove="Converters\**\*.cs" />
|
||||
<Compile Remove="Converters\\**\\*.cs" />
|
||||
<Compile Remove="Converters/Importers/**\*.cs" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user