using System.Linq; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using MongoDB.Bson; using StellaOps.Feedser.Models; using StellaOps.Feedser.Source.Ghsa; using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Ghsa.Internal; using StellaOps.Feedser.Source.Osv.Internal; using StellaOps.Feedser.Source.Osv; using StellaOps.Feedser.Source.Nvd; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Feedser.Storage.Mongo.Dtos; var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..")); var osvFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Feedser.Source.Osv.Tests", "Fixtures"); var ghsaFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Feedser.Source.Ghsa.Tests", "Fixtures"); var nvdFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Feedser.Source.Nvd.Tests", "Nvd", "Fixtures"); RewriteOsvFixtures(osvFixturesPath); RewriteSnapshotFixtures(osvFixturesPath); RewriteGhsaFixtures(osvFixturesPath); RewriteCreditParityFixtures(ghsaFixturesPath, nvdFixturesPath); return; void RewriteOsvFixtures(string fixturesPath) { var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-osv.json"); if (!File.Exists(rawPath)) { Console.WriteLine($"[FixtureUpdater] OSV raw fixture missing: {rawPath}"); return; } using var document = JsonDocument.Parse(File.ReadAllText(rawPath)); var advisories = new List(); foreach (var element in document.RootElement.EnumerateArray()) { var dto = JsonSerializer.Deserialize(element.GetRawText(), serializerOptions); if (dto is null) { continue; } var ecosystem = dto.Affected?.FirstOrDefault()?.Package?.Ecosystem ?? "unknown"; var uri = new Uri($"https://osv.dev/vulnerability/{dto.Id}"); var documentRecord = new DocumentRecord( Guid.NewGuid(), OsvConnectorPlugin.SourceName, uri.ToString(), DateTimeOffset.UtcNow, "fixture-sha", DocumentStatuses.PendingMap, "application/json", null, new Dictionary(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem, }, null, DateTimeOffset.UtcNow, null, null); var payload = BsonDocument.Parse(element.GetRawText()); var dtoRecord = new DtoRecord( Guid.NewGuid(), documentRecord.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, DateTimeOffset.UtcNow); var advisory = OsvMapper.Map(dto, documentRecord, dtoRecord, ecosystem); advisories.Add(advisory); } advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal)); var snapshot = SnapshotSerializer.ToSnapshot(advisories); File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.osv.json"), snapshot); Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.osv.json")}"); } void RewriteSnapshotFixtures(string fixturesPath) { var baselinePublished = new DateTimeOffset(2025, 1, 5, 12, 0, 0, TimeSpan.Zero); var baselineModified = new DateTimeOffset(2025, 1, 8, 6, 30, 0, TimeSpan.Zero); var baselineFetched = new DateTimeOffset(2025, 1, 8, 7, 0, 0, TimeSpan.Zero); var cases = new (string Ecosystem, string Purl, string PackageName, string SnapshotFile)[] { ("npm", "pkg:npm/%40scope%2Fleft-pad", "@scope/left-pad", "osv-npm.snapshot.json"), ("PyPI", "pkg:pypi/requests", "requests", "osv-pypi.snapshot.json"), }; foreach (var (ecosystem, purl, packageName, snapshotFile) in cases) { var dto = new OsvVulnerabilityDto { Id = $"OSV-2025-{ecosystem}-0001", Summary = $"{ecosystem} package vulnerability", Details = $"Detailed description for {ecosystem} package {packageName}.", Published = baselinePublished, Modified = baselineModified, Aliases = new[] { $"CVE-2025-11{ecosystem.Length}", $"GHSA-{ecosystem.Length}abc-{ecosystem.Length}def-{ecosystem.Length}ghi" }, Related = new[] { $"OSV-RELATED-{ecosystem}-42" }, References = new[] { new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/advisory", Type = "ADVISORY" }, new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/fix", Type = "FIX" }, }, Severity = new[] { new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, }, Affected = new[] { new OsvAffectedPackageDto { Package = new OsvPackageDto { Ecosystem = ecosystem, Name = packageName, Purl = purl, }, Ranges = new[] { new OsvRangeDto { Type = "SEMVER", Events = new[] { new OsvEventDto { Introduced = "0" }, new OsvEventDto { Fixed = "2.0.0" }, }, }, }, Versions = new[] { "1.0.0", "1.5.0" }, EcosystemSpecific = JsonDocument.Parse("{\"severity\":\"high\"}").RootElement.Clone(), }, }, DatabaseSpecific = JsonDocument.Parse("{\"source\":\"osv.dev\"}").RootElement.Clone(), }; var document = new DocumentRecord( Guid.NewGuid(), OsvConnectorPlugin.SourceName, $"https://osv.dev/vulnerability/{dto.Id}", baselineFetched, "fixture-sha", DocumentStatuses.PendingParse, "application/json", null, new Dictionary(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem }, null, baselineModified, null); var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, serializerOptions)); var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, baselineModified); var advisory = OsvMapper.Map(dto, document, dtoRecord, ecosystem); var snapshot = SnapshotSerializer.ToSnapshot(advisory); File.WriteAllText(Path.Combine(fixturesPath, snapshotFile), snapshot); Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, snapshotFile)}"); } } void RewriteGhsaFixtures(string fixturesPath) { var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-ghsa.json"); if (!File.Exists(rawPath)) { Console.WriteLine($"[FixtureUpdater] GHSA raw fixture missing: {rawPath}"); return; } JsonDocument document; try { document = JsonDocument.Parse(File.ReadAllText(rawPath)); } catch (JsonException ex) { Console.WriteLine($"[FixtureUpdater] Failed to parse GHSA raw fixture '{rawPath}': {ex.Message}"); return; } using (document) { var advisories = new List(); foreach (var element in document.RootElement.EnumerateArray()) { GhsaRecordDto dto; try { dto = GhsaRecordParser.Parse(Encoding.UTF8.GetBytes(element.GetRawText())); } catch (JsonException) { continue; } var uri = new Uri($"https://github.com/advisories/{dto.GhsaId}"); var documentRecord = new DocumentRecord( Guid.NewGuid(), GhsaConnectorPlugin.SourceName, uri.ToString(), DateTimeOffset.UtcNow, "fixture-sha", DocumentStatuses.PendingMap, "application/json", null, new Dictionary(StringComparer.Ordinal), null, DateTimeOffset.UtcNow, null, null); var advisory = GhsaMapper.Map(dto, documentRecord, DateTimeOffset.UtcNow); advisories.Add(advisory); } advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal)); var snapshot = SnapshotSerializer.ToSnapshot(advisories); File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.ghsa.json"), snapshot); Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.ghsa.json")}"); } } void RewriteCreditParityFixtures(string ghsaFixturesPath, string nvdFixturesPath) { Directory.CreateDirectory(ghsaFixturesPath); Directory.CreateDirectory(nvdFixturesPath); var advisoryKeyGhsa = "GHSA-credit-parity"; var advisoryKeyNvd = "CVE-2025-5555"; var recordedAt = new DateTimeOffset(2025, 10, 10, 15, 0, 0, TimeSpan.Zero); var published = new DateTimeOffset(2025, 10, 9, 18, 30, 0, TimeSpan.Zero); var modified = new DateTimeOffset(2025, 10, 10, 12, 0, 0, TimeSpan.Zero); AdvisoryCredit[] CreateCredits(string source) => [ CreateCredit("Alice Researcher", "reporter", new[] { "mailto:alice.researcher@example.com" }, source), CreateCredit("Bob Maintainer", "remediation_developer", new[] { "https://github.com/acme/bob-maintainer" }, source) ]; AdvisoryCredit CreateCredit(string displayName, string role, IReadOnlyList contacts, string source) { var provenance = new AdvisoryProvenance( source, "credit", $"{source}:{displayName.ToLowerInvariant().Replace(' ', '-')}", recordedAt, new[] { ProvenanceFieldMasks.Credits }); return new AdvisoryCredit(displayName, role, contacts, provenance); } AdvisoryReference[] CreateReferences(string sourceName, params (string Url, string Kind)[] entries) { if (entries is null || entries.Length == 0) { return Array.Empty(); } var references = new List(entries.Length); foreach (var entry in entries) { var provenance = new AdvisoryProvenance( sourceName, "reference", entry.Url, recordedAt, new[] { ProvenanceFieldMasks.References }); references.Add(new AdvisoryReference( entry.Url, entry.Kind, sourceTag: null, summary: null, provenance)); } return references.ToArray(); } Advisory CreateAdvisory( string sourceName, string advisoryKey, IEnumerable aliases, AdvisoryCredit[] credits, AdvisoryReference[] references, string documentValue) { var documentProvenance = new AdvisoryProvenance( sourceName, "document", documentValue, recordedAt, new[] { ProvenanceFieldMasks.Advisory }); var mappingProvenance = new AdvisoryProvenance( sourceName, "mapping", advisoryKey, recordedAt, new[] { ProvenanceFieldMasks.Advisory }); return new Advisory( advisoryKey, "Credit parity regression fixture", "Credit parity regression fixture", "en", published, modified, "moderate", exploitKnown: false, aliases, credits, references, Array.Empty(), Array.Empty(), new[] { documentProvenance, mappingProvenance }); } var ghsa = CreateAdvisory( "ghsa", advisoryKeyGhsa, new[] { advisoryKeyGhsa, advisoryKeyNvd }, CreateCredits("ghsa"), CreateReferences( "ghsa", ( $"https://github.com/advisories/{advisoryKeyGhsa}", "advisory"), ( "https://example.com/ghsa/patch", "patch")), $"security/advisories/{advisoryKeyGhsa}"); var osv = CreateAdvisory( OsvConnectorPlugin.SourceName, advisoryKeyGhsa, new[] { advisoryKeyGhsa, advisoryKeyNvd }, CreateCredits(OsvConnectorPlugin.SourceName), CreateReferences( OsvConnectorPlugin.SourceName, ( $"https://github.com/advisories/{advisoryKeyGhsa}", "advisory"), ( $"https://osv.dev/vulnerability/{advisoryKeyGhsa}", "advisory")), $"https://osv.dev/vulnerability/{advisoryKeyGhsa}"); var nvd = CreateAdvisory( NvdConnectorPlugin.SourceName, advisoryKeyNvd, new[] { advisoryKeyNvd, advisoryKeyGhsa }, CreateCredits(NvdConnectorPlugin.SourceName), CreateReferences( NvdConnectorPlugin.SourceName, ( $"https://services.nvd.nist.gov/vuln/detail/{advisoryKeyNvd}", "advisory"), ( "https://example.com/nvd/reference", "report")), $"https://services.nvd.nist.gov/vuln/detail/{advisoryKeyNvd}"); var ghsaSnapshot = SnapshotSerializer.ToSnapshot(ghsa); var osvSnapshot = SnapshotSerializer.ToSnapshot(osv); var nvdSnapshot = SnapshotSerializer.ToSnapshot(nvd); File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.ghsa.json"), ghsaSnapshot); File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.osv.json"), osvSnapshot); File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.nvd.json"), nvdSnapshot); File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.ghsa.json"), ghsaSnapshot); File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.osv.json"), osvSnapshot); File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.nvd.json"), nvdSnapshot); Console.WriteLine($"[FixtureUpdater] Updated credit parity fixtures under {ghsaFixturesPath} and {nvdFixturesPath}"); }