This commit is contained in:
master
2025-10-29 19:24:20 +02:00
parent 86f606a115
commit fac626db8d
41 changed files with 2134 additions and 168 deletions

View File

@@ -9,4 +9,4 @@
|FEEDCONN-CCCS-02-006 Observability & documentation|DevEx|Docs|**DONE (2025-10-15)** Added `CccsDiagnostics` meter (fetch/parse/map counters), enriched connector logs with document counts, and published `docs/ops/concelier-cccs-operations.md` covering config, telemetry, and sanitiser guidance.|
|FEEDCONN-CCCS-02-007 Historical advisory harvesting plan|BE-Conn-CCCS|Research|**DONE (2025-10-15)** Measured `/api/cccs/threats/v1/get` inventory (~5.1k rows/lang; earliest 2018-06-08), documented backfill workflow + language split strategy, and linked the runbook for Offline Kit execution.|
|FEEDCONN-CCCS-02-008 Raw DOM parsing refinement|BE-Conn-CCCS|Source.Common|**DONE (2025-10-15)** Parser now walks unsanitised DOM (heading + nested list coverage), sanitizer keeps `<h#>`/`section` nodes, and regression fixtures/tests assert EN/FR list handling + preserved HTML structure.|
|FEEDCONN-CCCS-02-009 Normalized versions rollout (Oct 2025)|BE-Conn-CCCS|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-21)** Implement trailing-version split helper per Merge guidance (see `../Merge/RANGE_PRIMITIVES_COORDINATION.md` “Helper snippets”) to emit `NormalizedVersions` via `SemVerRangeRuleBuilder`; refresh mapper tests/fixtures to assert provenance notes (`cccs:{serial}:{index}`) and confirm merge counters drop.|
|FEEDCONN-CCCS-02-009 Normalized versions rollout (Oct 2025)|BE-Conn-CCCS|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-21)** Implement trailing-version split helper per Merge guidance (see `../Merge/RANGE_PRIMITIVES_COORDINATION.md` “Helper snippets”) to emit `NormalizedVersions` via `SemVerRangeRuleBuilder`; refresh mapper tests/fixtures to assert provenance notes (`cccs:{serial}:{index}`) and confirm merge counters drop.<br>2025-10-29: See `docs/dev/normalized-rule-recipes.md` for ready-made helper + regex snippet; wire into `BuildPackages` and update fixtures with `UPDATE_CCCS_FIXTURES=1`.|

View File

@@ -10,4 +10,4 @@
|FEEDCONN-CERTBUND-02-007 Feed history & locale assessment|BE-Conn-CERTBUND|Research|**DONE (2025-10-15)** Measured RSS retention (~6days/≈250 items), captured connector-driven backfill guidance in the runbook, and aligned locale guidance (preserve `language=de`, Docs glossary follow-up). **Next:** coordinate with Tools to land the state-seeding helper so scripted backfills replace manual Mongo tweaks.|
|FEEDCONN-CERTBUND-02-008 Session bootstrap & cookie strategy|BE-Conn-CERTBUND|Source.Common|**DONE (2025-10-14)** Feed client primes the portal session (cookie container via `SocketsHttpHandler`), shares cookies across detail requests, and documents bootstrap behaviour in options (`PortalBootstrapUri`).|
|FEEDCONN-CERTBUND-02-009 Offline Kit export packaging|BE-Conn-CERTBUND, Docs|Offline Kit|**DONE (2025-10-17)** Added `tools/certbund_offline_snapshot.py` to capture search/export JSON, emit deterministic manifests + SHA files, and refreshed docs (`docs/ops/concelier-certbund-operations.md`, `docs/24_OFFLINE_KIT.md`) with offline-kit instructions and manifest layout guidance. Seed data README/ignore rules cover local snapshot hygiene.|
|FEEDCONN-CERTBUND-02-010 Normalized range translator|BE-Conn-CERTBUND|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-22)** Translate `product.Versions` phrases (e.g., `2023.1 bis 2024.2`, `alle`) into comparator strings for `SemVerRangeRuleBuilder`, emit `NormalizedVersions` with `certbund:{advisoryId}:{vendor}` provenance, and extend tests/README with localisation notes.|
|FEEDCONN-CERTBUND-02-010 Normalized range translator|BE-Conn-CERTBUND|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-22)** Translate `product.Versions` phrases (e.g., `2023.1 bis 2024.2`, `alle`) into comparator strings for `SemVerRangeRuleBuilder`, emit `NormalizedVersions` with `certbund:{advisoryId}:{vendor}` provenance, and extend tests/README with localisation notes.<br>2025-10-29: `docs/dev/normalized-rule-recipes.md` §3 includes regex starter for German “bis” phrases—integrate into mapper and refresh fixtures via `UPDATE_CERTBUND_FIXTURES=1`.|

View File

@@ -13,4 +13,4 @@
|Express unaffected/investigation statuses without overloading range fields|BE-Conn-RH|Models|**DONE** Introduced AffectedPackageStatus collection and updated mapper/tests.|
|Reference dedupe & ordering in mapper|BE-Conn-RH|Models|DONE mapper consolidates by URL, merges metadata, deterministic ordering validated in tests.|
|Hydra summary fetch through SourceFetchService|BE-Conn-RH|Source.Common|DONE summary pages now fetched via SourceFetchService with cache + conditional headers.|
|Fixture validation sweep|QA|None|**DOING (2025-10-19)** Prereqs confirmed none; continuing RHSA fixture regeneration and diff review alongside mapper provenance updates.|
|Fixture validation sweep|QA|None|**DOING (2025-10-19)** Prereqs confirmed none; continuing RHSA fixture regeneration and diff review alongside mapper provenance updates.<br>2025-10-29: Added `scripts/update-redhat-fixtures.sh` to regenerate golden snapshots with `UPDATE_GOLDENS=1`; run it before reviews to capture CSAF contract deltas.|

View File

@@ -12,4 +12,4 @@
|FEEDCONN-ICSCISA-02-009 GovDelivery credential onboarding|Ops, BE-Conn-ICS-CISA|Ops|**DONE (2025-10-14)** GovDelivery onboarding runbook captured in `docs/ops/concelier-icscisa-operations.md`; secret vault path and Offline Kit handling documented.|
|FEEDCONN-ICSCISA-02-010 Mitigation & SemVer polish|BE-Conn-ICS-CISA|02-003, 02-004|**DONE (2025-10-16)** Attachment + mitigation references now land as expected and SemVer primitives carry exact values; end-to-end suite green (see `HANDOVER.md`).|
|FEEDCONN-ICSCISA-02-011 Docs & telemetry refresh|DevEx|02-006|**DONE (2025-10-16)** Ops documentation refreshed (attachments, SemVer validation, proxy knobs) and telemetry notes verified.|
|FEEDCONN-ICSCISA-02-012 Normalized version decision|BE-Conn-ICS-CISA|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-23)** Promote existing `SemVerPrimitive` exact values into `NormalizedVersions` via `.ToNormalizedVersionRule("ics-cisa:{advisoryId}:{product}")`, add regression coverage, and open Models ticket if non-SemVer firmware requires a new scheme.|
|FEEDCONN-ICSCISA-02-012 Normalized version decision|BE-Conn-ICS-CISA|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-23)** Promote existing `SemVerPrimitive` exact values into `NormalizedVersions` via `.ToNormalizedVersionRule("ics-cisa:{advisoryId}:{product}")`, add regression coverage, and open Models ticket if non-SemVer firmware requires a new scheme.<br>2025-10-29: Follow `docs/dev/normalized-rule-recipes.md` §2 to call `ToNormalizedVersionRule` and ensure mixed firmware strings log a Models ticket when regex extraction fails.|

View File

@@ -9,4 +9,4 @@
|FEEDCONN-CISCO-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** Cisco diagnostics counters exposed and ops runbook updated with telemetry guidance (`docs/ops/concelier-cisco-operations.md`).|
|FEEDCONN-CISCO-02-007 API selection decision memo|BE-Conn-Cisco|Research|**DONE (2025-10-11)** Drafted decision matrix: openVuln (structured/delta filters, OAuth throttle) vs RSS (delayed/minimal metadata). Pending OAuth onboarding (`FEEDCONN-CISCO-02-008`) before final recommendation circulated.|
|FEEDCONN-CISCO-02-008 OAuth client provisioning|Ops, BE-Conn-Cisco|Ops|**DONE (2025-10-14)** `docs/ops/concelier-cisco-operations.md` documents OAuth provisioning/rotation, quotas, and Offline Kit distribution guidance.|
|FEEDCONN-CISCO-02-009 Normalized SemVer promotion|BE-Conn-Cisco|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-21)** Use helper from `../Merge/RANGE_PRIMITIVES_COORDINATION.md` to convert `SemVerPrimitive` outputs into `NormalizedVersionRule` with provenance (`cisco:{productId}`), update mapper/tests, and confirm merge normalized-rule counters drop.|
|FEEDCONN-CISCO-02-009 Normalized SemVer promotion|BE-Conn-Cisco|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-21)** Use helper from `../Merge/RANGE_PRIMITIVES_COORDINATION.md` to convert `SemVerPrimitive` outputs into `NormalizedVersionRule` with provenance (`cisco:{productId}`), update mapper/tests, and confirm merge normalized-rule counters drop.<br>2025-10-29: Follow `docs/dev/normalized-rule-recipes.md` §2 to convert existing primitives (`CiscoMapper`) and document provenance in fixtures (`UPDATE_CISCO_FIXTURES=1`).|

View File

@@ -336,46 +336,67 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
string.IsNullOrWhiteSpace(content.Encoding) ? null : content.Encoding.Trim());
}
private static RawIdentifiers NormalizeIdentifiers(RawIdentifiers identifiers)
{
var normalizedAliases = identifiers.Aliases
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
.Select(static alias => alias.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
return new RawIdentifiers(
normalizedAliases,
identifiers.PrimaryId?.Trim() ?? string.Empty);
}
private static RawLinkset NormalizeLinkset(RawLinkset linkset)
{
return new RawLinkset
{
Aliases = NormalizeStringArray(linkset.Aliases, StringComparer.OrdinalIgnoreCase),
PackageUrls = NormalizeStringArray(linkset.PackageUrls, StringComparer.Ordinal),
Cpes = NormalizeStringArray(linkset.Cpes, StringComparer.Ordinal),
References = NormalizeReferences(linkset.References),
ReconciledFrom = NormalizeStringArray(linkset.ReconciledFrom, StringComparer.Ordinal),
Notes = linkset.Notes ?? ImmutableDictionary<string, string>.Empty,
};
}
private static ImmutableArray<string> NormalizeStringArray(ImmutableArray<string> values, StringComparer comparer)
{
if (values.IsDefaultOrEmpty)
{
return EmptyArray;
}
return values
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value.Trim())
.Distinct(comparer)
.OrderBy(static value => value, comparer)
.ToImmutableArray();
}
private static RawIdentifiers NormalizeIdentifiers(RawIdentifiers identifiers)
{
var aliases = identifiers.Aliases;
if (!aliases.IsDefaultOrEmpty)
{
var builder = ImmutableArray.CreateBuilder<string>(aliases.Length);
foreach (var alias in aliases)
{
if (string.IsNullOrWhiteSpace(alias))
{
continue;
}
builder.Add(alias.Trim());
}
aliases = builder.ToImmutable();
}
else
{
aliases = ImmutableArray<string>.Empty;
}
return new RawIdentifiers(
aliases,
identifiers.PrimaryId?.Trim() ?? string.Empty);
}
private static RawLinkset NormalizeLinkset(RawLinkset linkset)
{
return new RawLinkset
{
Aliases = NormalizeStringArray(linkset.Aliases),
PackageUrls = NormalizeStringArray(linkset.PackageUrls),
Cpes = NormalizeStringArray(linkset.Cpes),
References = NormalizeReferences(linkset.References),
ReconciledFrom = NormalizeStringArray(linkset.ReconciledFrom),
Notes = linkset.Notes ?? ImmutableDictionary<string, string>.Empty,
};
}
private static ImmutableArray<string> NormalizeStringArray(ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>(values.Length);
foreach (var value in values)
{
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(value.Trim());
}
return builder.ToImmutable();
}
private static ImmutableArray<RawReference> NormalizeReferences(ImmutableArray<RawReference> references)
{

View File

@@ -12,8 +12,9 @@
| CONCELIER-CORE-AOC-19-003 `Idempotent append-only upsert` | DONE (2025-10-28) | Concelier Core Guild | CONCELIER-STORE-AOC-19-002 | Implement idempotent upsert path using `(vendor, upstreamId, contentHash, tenant)` key, emitting supersedes pointers for new revisions and preventing duplicate inserts. |
> 2025-10-28: Advisory raw ingestion now strips client-supplied supersedes hints, logs ignored pointers, and surfaces repository-supplied supersedes identifiers; service tests cover duplicate handling and append-only semantics.
> Docs alignment (2025-10-26): Deployment guide + observability guide describe supersedes metrics; ensure implementation emits `aoc_violation_total` on failure.
| CONCELIER-CORE-AOC-19-004 `Remove ingestion normalization` | DOING (2025-10-28) | Concelier Core Guild | CONCELIER-CORE-AOC-19-002, POLICY-AOC-19-003 | Strip normalization/dedup/severity logic from ingestion pipelines, delegate derived computations to Policy Engine, and update exporters/tests to consume raw documents only. |
> Docs alignment (2025-10-26): Architecture overview emphasises policy-only derivation; coordinate with Policy Engine guild for rollout.
| CONCELIER-CORE-AOC-19-004 `Remove ingestion normalization` | DOING (2025-10-28) | Concelier Core Guild | CONCELIER-CORE-AOC-19-002, POLICY-AOC-19-003 | Strip normalization/dedup/severity logic from ingestion pipelines, delegate derived computations to Policy Engine, and update exporters/tests to consume raw documents only. |
> Docs alignment (2025-10-26): Architecture overview emphasises policy-only derivation; coordinate with Policy Engine guild for rollout.
> 2025-10-29: `AdvisoryRawService` now preserves upstream alias/linkset ordering (trim-only) and updated AOC documentation reflects the behaviour; follow-up to ensure policy consumers handle duplicates remains open.
| CONCELIER-CORE-AOC-19-013 `Authority tenant scope smoke coverage` | TODO | Concelier Core Guild | AUTH-AOC-19-002 | Extend Concelier smoke/e2e fixtures to configure `requiredTenants` and assert cross-tenant rejection with updated Authority tokens. | Coordinate deliverable so Authority docs (`AUTH-AOC-19-003`) can close once tests are in place. |
## Policy Engine v2
@@ -27,10 +28,12 @@
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| CONCELIER-GRAPH-21-001 `SBOM projection enrichment` | BLOCKED (2025-10-27) | Concelier Core Guild, Cartographer Guild | CONCELIER-POLICY-20-002, CARTO-GRAPH-21-002 | Extend SBOM normalization to emit full relationship graph (depends_on/contains/provides), scope tags, entrypoint annotations, and component metadata required by Cartographer. |
> 2025-10-27: Waiting on policy-driven linkset enrichment (`CONCELIER-POLICY-20-002`) and Cartographer API contract (`CARTO-GRAPH-21-002`) to define required relationship payloads. Without those schemas the projection changes cannot be implemented deterministically.
| CONCELIER-GRAPH-21-002 `Change events` | BLOCKED (2025-10-27) | Concelier Core Guild, Scheduler Guild | CONCELIER-GRAPH-21-001 | Publish change events (new SBOM version, relationship delta) for Cartographer build queue; ensure events include tenant/context metadata. |
> 2025-10-27: Depends on `CONCELIER-GRAPH-21-001`; event schema hinges on finalized projection output and Cartographer webhook contract, both pending.
| CONCELIER-GRAPH-21-001 `SBOM projection enrichment` | BLOCKED (2025-10-27) | Concelier Core Guild, Cartographer Guild | CONCELIER-POLICY-20-002, CARTO-GRAPH-21-002 | Extend SBOM normalization to emit full relationship graph (depends_on/contains/provides), scope tags, entrypoint annotations, and component metadata required by Cartographer. |
> 2025-10-27: Waiting on policy-driven linkset enrichment (`CONCELIER-POLICY-20-002`) and Cartographer API contract (`CARTO-GRAPH-21-002`) to define required relationship payloads. Without those schemas the projection changes cannot be implemented deterministically.
> 2025-10-29: Cross-guild handshake captured in `docs/dev/cartographer-graph-handshake.md`; begin drafting enrichment plan once Cartographer ships the inspector schema/query patterns.
| CONCELIER-GRAPH-21-002 `Change events` | BLOCKED (2025-10-27) | Concelier Core Guild, Scheduler Guild | CONCELIER-GRAPH-21-001 | Publish change events (new SBOM version, relationship delta) for Cartographer build queue; ensure events include tenant/context metadata. |
> 2025-10-27: Depends on `CONCELIER-GRAPH-21-001`; event schema hinges on finalized projection output and Cartographer webhook contract, both pending.
> 2025-10-29: Action item from handshake doc — prepare sample `sbom.relationship.changed` payload + replay notes once schema lands; coordinate with Scheduler for queue semantics.
## Link-Not-Merge v1

View File

@@ -92,6 +92,7 @@ Until these blocks land, connectors should stage changes behind a feature flag o
## Tracking & follow-up
- Track due dates above; if a connector slips past its deadline, flag in `#concelier-merge` stand-up and open a blocker ticket referencing FEEDMERGE-COORD-02-900.
- Capture connector progress updates in stand-ups twice per week; link PRs/issues back to this document and the rollout dashboard (`docs/dev/normalized_versions_rollout.md`).
- Monitor merge counters `concelier.merge.normalized_rules` and `concelier.merge.normalized_rules_missing` to spot advisories that still lack normalized arrays after precedence merge.
- Monitor merge counters `concelier.merge.normalized_rules` and `concelier.merge.normalized_rules_missing` to spot advisories that still lack normalized arrays after precedence merge.
- Precedence merge emits `Normalized version rules missing` warnings (source + package type) whenever we encounter ranges without normalized output—watch CI/staging logs for those signals to prioritise backlog fixes.
- When a connector is ready to emit normalized rules, update its module `TASKS.md` status and ping Merge in `#concelier-merge` with fixture diff screenshots.
- If new schemes or comparer logic is required (e.g., Cisco IOS), open a Models issue referencing `FEEDMODELS-SCHEMA-02-900` before implementing.

View File

@@ -162,7 +162,7 @@ public sealed class AdvisoryPrecedenceMerger
.ToArray();
var packageResult = _packageResolver.Merge(ordered.SelectMany(entry => entry.Advisory.AffectedPackages));
RecordNormalizedRuleMetrics(packageResult.Packages);
RecordNormalizedRuleMetrics(advisoryKey, packageResult.Packages);
var affectedPackages = packageResult.Packages;
var cvssMetrics = ordered
.SelectMany(entry => entry.Advisory.CvssMetrics)
@@ -217,13 +217,16 @@ public sealed class AdvisoryPrecedenceMerger
return new PrecedenceMergeResult(merged, conflicts);
}
private static void RecordNormalizedRuleMetrics(IReadOnlyList<AffectedPackage> packages)
private void RecordNormalizedRuleMetrics(string advisoryKey, IReadOnlyList<AffectedPackage> packages)
{
if (packages.Count == 0)
{
return;
}
var missingSources = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var missingPackageTypes = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var package in packages)
{
var packageType = package.Type ?? string.Empty;
@@ -249,8 +252,41 @@ public sealed class AdvisoryPrecedenceMerger
};
MissingNormalizedRuleCounter.Add(1, tags);
if (package.Provenance.Length > 0)
{
foreach (var provenance in package.Provenance)
{
if (string.IsNullOrWhiteSpace(provenance.Source))
{
continue;
}
if (!string.Equals(provenance.Source, "merge", StringComparison.OrdinalIgnoreCase))
{
missingSources.Add(provenance.Source);
}
}
}
if (!string.IsNullOrWhiteSpace(packageType))
{
missingPackageTypes.Add(packageType);
}
}
}
if (missingSources.Count > 0)
{
var sources = string.Join(",", missingSources.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase));
var packageTypes = string.Join(",", missingPackageTypes.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase));
_logger.LogWarning(
"Normalized version rules missing for {AdvisoryKey}; sources={Sources}; packageTypes={PackageTypes}",
advisoryKey,
sources,
packageTypes);
}
}
private string? PickString(IEnumerable<AdvisoryEntry> ordered, Func<Advisory, string?> selector)

View File

@@ -15,14 +15,14 @@
|FEEDMERGE-QA-04-001 End-to-end conflict regression suite|QA|Merge|DONE `AdvisoryMergeServiceTests.MergeAsync_AppliesCanonicalRulesAndPersistsDecisions` exercises GHSA/NVD/OSV conflict path and merge-event analytics. **Reminder:** QA to sync with connector teams once new fixture triples land.|
|Override audit logging|BE-Merge|Observability|DONE override audits now emit structured logs plus bounded-tag metrics suitable for prod telemetry.|
|Configurable precedence table|BE-Merge|Architecture|DONE precedence options bind via concelier:merge:precedence:ranks with docs/tests covering operator workflow.|
|Range primitives backlog|BE-Merge|Connector WGs|**DOING** Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical RangePrimitives with provenance tags; track progress/fixtures here.<br>2025-10-11: Storage alignment notes + sample normalized rule JSON now captured in `RANGE_PRIMITIVES_COORDINATION.md` (see “Storage alignment quick reference”).<br>2025-10-11 18:45Z: GHSA normalized rules landed; OSV connector picked up next for rollout.<br>2025-10-11 21:10Z: `docs/dev/merge_semver_playbook.md` Section 8 now documents the persisted Mongo projection (SemVer + NEVRA) for connector reviewers.<br>2025-10-11 21:30Z: Added `docs/dev/normalized_versions_rollout.md` dashboard to centralize connector status and upcoming milestones.<br>2025-10-11 21:55Z: Merge now emits `concelier.merge.normalized_rules*` counters and unions connector-provided normalized arrays; see new test coverage in `AdvisoryPrecedenceMergerTests.Merge_RecordsNormalizedRuleMetrics`.<br>2025-10-12 17:05Z: CVE + KEV normalized rule verification complete; OSV parity fixtures revalidated—downstream parity/monitoring tasks may proceed.<br>2025-10-19 14:35Z: Prerequisites reviewed (none outstanding); FEEDMERGE-COORD-02-900 remains in DOING with connector follow-ups unchanged.<br>2025-10-19 15:25Z: Refreshed `RANGE_PRIMITIVES_COORDINATION.md` matrix + added targeted follow-ups (Cccs, CertBund, ICS-CISA, Kisa, Vndr.Cisco) with delivery dates 2025-10-21 → 2025-10-25; monitoring merge counters for regression.|
|Range primitives backlog|BE-Merge|Connector WGs|**DOING** Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical RangePrimitives with provenance tags; track progress/fixtures here.<br>2025-10-11: Storage alignment notes + sample normalized rule JSON now captured in `RANGE_PRIMITIVES_COORDINATION.md` (see “Storage alignment quick reference”).<br>2025-10-11 18:45Z: GHSA normalized rules landed; OSV connector picked up next for rollout.<br>2025-10-11 21:10Z: `docs/dev/merge_semver_playbook.md` Section 8 now documents the persisted Mongo projection (SemVer + NEVRA) for connector reviewers.<br>2025-10-11 21:30Z: Added `docs/dev/normalized_versions_rollout.md` dashboard to centralize connector status and upcoming milestones.<br>2025-10-11 21:55Z: Merge now emits `concelier.merge.normalized_rules*` counters and unions connector-provided normalized arrays; see new test coverage in `AdvisoryPrecedenceMergerTests.Merge_RecordsNormalizedRuleMetrics`.<br>2025-10-12 17:05Z: CVE + KEV normalized rule verification complete; OSV parity fixtures revalidated—downstream parity/monitoring tasks may proceed.<br>2025-10-19 14:35Z: Prerequisites reviewed (none outstanding); FEEDMERGE-COORD-02-900 remains in DOING with connector follow-ups unchanged.<br>2025-10-19 15:25Z: Refreshed `RANGE_PRIMITIVES_COORDINATION.md` matrix + added targeted follow-ups (Cccs, CertBund, ICS-CISA, Kisa, Vndr.Cisco) with delivery dates 2025-10-21 → 2025-10-25; monitoring merge counters for regression.<br>2025-10-29: Added merge-time warnings highlighting sources/package types when ranges emit without normalized rules to accelerate backlog triage.|
|Range primitives backlog|BE-Merge|Connector WGs|**DOING** Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical RangePrimitives with provenance tags; track progress/fixtures here.<br>2025-10-11: Storage alignment notes + sample normalized rule JSON now captured in `RANGE_PRIMITIVES_COORDINATION.md` (see “Storage alignment quick reference”).<br>2025-10-11 18:45Z: GHSA normalized rules landed; OSV connector picked up next for rollout.<br>2025-10-11 21:10Z: `docs/dev/merge_semver_playbook.md` Section 8 now documents the persisted Mongo projection (SemVer + NEVRA) for connector reviewers.<br>2025-10-11 21:30Z: Added `docs/dev/normalized_versions_rollout.md` dashboard to centralize connector status and upcoming milestones.<br>2025-10-11 21:55Z: Merge now emits `concelier.merge.normalized_rules*` counters and unions connector-provided normalized arrays; see new test coverage in `AdvisoryPrecedenceMergerTests.Merge_RecordsNormalizedRuleMetrics`.<br>2025-10-12 17:05Z: CVE + KEV normalized rule verification complete; OSV parity fixtures revalidated—downstream parity/monitoring tasks may proceed.<br>2025-10-19 14:35Z: Prerequisites reviewed (none outstanding); FEEDMERGE-COORD-02-900 remains in DOING with connector follow-ups unchanged.<br>2025-10-19 15:25Z: Refreshed `RANGE_PRIMITIVES_COORDINATION.md` matrix + added targeted follow-ups (Cccs, CertBund, ICS-CISA, Kisa, Vndr.Cisco) with delivery dates 2025-10-21 → 2025-10-25; monitoring merge counters for regression.<br>2025-10-20 19:30Z: Coordination matrix + rollout dashboard updated with current connector statuses and due dates; flagged Slack escalation plan if Cccs/Cisco miss 2025-10-21 and documented Acsc kickoff window for 2025-10-24.|
|Merge pipeline parity for new advisory fields|BE-Merge|Models, Core|DONE (2025-10-15) merge service now surfaces description/CWE/canonical metric decisions with updated metrics/tests.|
|Connector coordination for new advisory fields|Connector Leads, BE-Merge|Models, Core|**DONE (2025-10-15)** GHSA, NVD, and OSV connectors now emit advisory descriptions, CWE weaknesses, and canonical metric ids. Fixtures refreshed (GHSA connector regression suite, `conflict-nvd.canonical.json`, OSV parity snapshots) and completion recorded in coordination log.|
|FEEDMERGE-ENGINE-07-001 Conflict sets & explainers|BE-Merge|FEEDSTORAGE-DATA-07-001|**DONE (2025-10-20)** Merge surfaces conflict explainers with replay hashes via `MergeConflictSummary`; API exposes structured payloads and integration tests cover deterministic `asOf` hashes.|
> Remark (2025-10-20): `AdvisoryMergeService` now returns conflict summaries with deterministic hashes; WebService replay endpoint emits typed explainers verified by new tests.
|FEEDMERGE-COORD-02-901 Connector deadline check-ins|BE-Merge|FEEDMERGE-COORD-02-900|**TODO (due 2025-10-21)** Confirm Cccs/Cisco normalized-rule branches land, capture `concelier.merge.normalized_rules*` counter screenshots, and update coordination docs with the results.|
|FEEDMERGE-COORD-02-902 ICS-CISA normalized-rule decision support|BE-Merge, Models|FEEDMERGE-COORD-02-900|**TODO (due 2025-10-23)** Review ICS-CISA sample advisories, confirm SemVer reuse vs new firmware scheme, pre-stage Models ticket template, and document outcome in coordination docs + tracker files.|
|FEEDMERGE-COORD-02-901 Connector deadline check-ins|BE-Merge|FEEDMERGE-COORD-02-900|**TODO (due 2025-10-21)** Confirm Cccs/Cisco normalized-rule branches land, capture `concelier.merge.normalized_rules*` counter screenshots, and update coordination docs with the results.<br>2025-10-29: Merge now emits `Normalized version rules missing...` warnings (see `docs/dev/normalized-rule-recipes.md` §4); include zero-warning excerpt plus Grafana counter snapshot when closing this task.|
|FEEDMERGE-COORD-02-902 ICS-CISA normalized-rule decision support|BE-Merge, Models|FEEDMERGE-COORD-02-900|**TODO (due 2025-10-23)** Review ICS-CISA sample advisories, confirm SemVer reuse vs new firmware scheme, pre-stage Models ticket template, and document outcome in coordination docs + tracker files.<br>2025-10-29: Recipes doc (§2§3) outlines SemVer promotion + fallback logging—attach decision summary + log sample when handing off to Models.|
|FEEDMERGE-COORD-02-903 KISA firmware scheme review|BE-Merge, Models|FEEDMERGE-COORD-02-900|**TODO (due 2025-10-24)** Pair with KISA team on proposed firmware scheme (`kisa.build` or variant), ensure builder alignment, open Models ticket if required, and log decision in coordination docs + tracker files.|
## Link-Not-Merge v1 Transition

View File

@@ -0,0 +1,212 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Common.State;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
namespace StellaOps.Concelier.Connector.Common.Tests;
public sealed class SourceStateSeedProcessorTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly MongoClient _client;
private readonly IMongoDatabase _database;
private readonly DocumentStore _documentStore;
private readonly RawDocumentStorage _rawStorage;
private readonly MongoSourceStateRepository _stateRepository;
private readonly FakeTimeProvider _timeProvider;
public SourceStateSeedProcessorTests()
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
_client = new MongoClient(_runner.ConnectionString);
_database = _client.GetDatabase($"source-state-seed-{Guid.NewGuid():N}");
_documentStore = new DocumentStore(_database, NullLogger<DocumentStore>.Instance);
_rawStorage = new RawDocumentStorage(_database);
_stateRepository = new MongoSourceStateRepository(_database, NullLogger<MongoSourceStateRepository>.Instance);
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 28, 12, 0, 0, TimeSpan.Zero));
}
[Fact]
public async Task ProcessAsync_PersistsDocumentsAndUpdatesCursor()
{
var processor = CreateProcessor();
var documentId = Guid.NewGuid();
var specification = new SourceStateSeedSpecification
{
Source = "vndr.test",
Documents = new[]
{
new SourceStateSeedDocument
{
DocumentId = documentId,
Uri = "https://example.test/advisories/ADV-1",
Content = Encoding.UTF8.GetBytes("{\"id\":\"ADV-1\"}"),
ContentType = "application/json",
Headers = new Dictionary<string, string> { ["X-Test"] = "true" },
Metadata = new Dictionary<string, string> { ["test.meta"] = "value" },
FetchedAt = _timeProvider.GetUtcNow().AddMinutes(-5),
AddToPendingDocuments = true,
AddToPendingMappings = true,
KnownIdentifiers = new[] { "ADV-1" },
}
},
Cursor = new SourceStateSeedCursor
{
LastModifiedCursor = _timeProvider.GetUtcNow().AddDays(-1),
LastFetchAt = _timeProvider.GetUtcNow().AddMinutes(-10),
Additional = new Dictionary<string, string> { ["custom"] = "value" },
},
KnownAdvisories = new[] { "ADV-0" },
};
var result = await processor.ProcessAsync(specification, CancellationToken.None);
Assert.Equal(1, result.DocumentsProcessed);
Assert.Single(result.PendingDocumentIds);
Assert.Contains(documentId, result.PendingDocumentIds);
Assert.Single(result.PendingMappingIds);
Assert.Contains(documentId, result.PendingMappingIds);
Assert.Equal(2, result.KnownAdvisoriesAdded.Count);
Assert.Contains("ADV-0", result.KnownAdvisoriesAdded);
Assert.Contains("ADV-1", result.KnownAdvisoriesAdded);
Assert.Equal(_timeProvider.GetUtcNow(), result.CompletedAt);
var storedDocument = await _documentStore.FindBySourceAndUriAsync(
"vndr.test",
"https://example.test/advisories/ADV-1",
CancellationToken.None);
Assert.NotNull(storedDocument);
Assert.Equal(documentId, storedDocument!.Id);
Assert.Equal("application/json", storedDocument.ContentType);
Assert.Equal(DocumentStatuses.PendingParse, storedDocument.Status);
Assert.NotNull(storedDocument.GridFsId);
Assert.NotNull(storedDocument.Headers);
Assert.Equal("true", storedDocument.Headers!["X-Test"]);
Assert.NotNull(storedDocument.Metadata);
Assert.Equal("value", storedDocument.Metadata!["test.meta"]);
var filesCollection = _database.GetCollection<BsonDocument>("documents.files");
var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
Assert.Equal(1, fileCount);
var state = await _stateRepository.TryGetAsync("vndr.test", CancellationToken.None);
Assert.NotNull(state);
Assert.Equal(_timeProvider.GetUtcNow().UtcDateTime, state!.LastSuccess);
var cursor = state.Cursor;
var pendingDocs = cursor["pendingDocuments"].AsBsonArray.Select(v => Guid.Parse(v.AsString)).ToList();
Assert.Contains(documentId, pendingDocs);
var pendingMappings = cursor["pendingMappings"].AsBsonArray.Select(v => Guid.Parse(v.AsString)).ToList();
Assert.Contains(documentId, pendingMappings);
var knownAdvisories = cursor["knownAdvisories"].AsBsonArray.Select(v => v.AsString).ToList();
Assert.Contains("ADV-0", knownAdvisories);
Assert.Contains("ADV-1", knownAdvisories);
Assert.Equal(_timeProvider.GetUtcNow().UtcDateTime, cursor["lastSeededAt"].ToUniversalTime());
Assert.Equal("value", cursor["custom"].AsString);
}
[Fact]
public async Task ProcessAsync_ReplacesExistingDocumentAndCleansPreviousRawPayload()
{
var processor = CreateProcessor();
var documentId = Guid.NewGuid();
var initialSpecification = new SourceStateSeedSpecification
{
Source = "vndr.test",
Documents = new[]
{
new SourceStateSeedDocument
{
DocumentId = documentId,
Uri = "https://example.test/advisories/ADV-2",
Content = Encoding.UTF8.GetBytes("{\"id\":\"ADV-2\",\"rev\":1}"),
ContentType = "application/json",
AddToPendingDocuments = true,
}
},
KnownAdvisories = new[] { "ADV-2" },
};
await processor.ProcessAsync(initialSpecification, CancellationToken.None);
var existingRecord = await _documentStore.FindBySourceAndUriAsync(
"vndr.test",
"https://example.test/advisories/ADV-2",
CancellationToken.None);
Assert.NotNull(existingRecord);
var previousGridId = existingRecord!.GridFsId;
Assert.NotNull(previousGridId);
var filesCollection = _database.GetCollection<BsonDocument>("documents.files");
var initialFiles = await filesCollection.Find(FilterDefinition<BsonDocument>.Empty).ToListAsync();
Assert.Single(initialFiles);
var updatedSpecification = new SourceStateSeedSpecification
{
Source = "vndr.test",
Documents = new[]
{
new SourceStateSeedDocument
{
DocumentId = documentId,
Uri = "https://example.test/advisories/ADV-2",
Content = Encoding.UTF8.GetBytes("{\"id\":\"ADV-2\",\"rev\":2}"),
ContentType = "application/json",
AddToPendingDocuments = true,
}
}
};
var secondResult = await processor.ProcessAsync(updatedSpecification, CancellationToken.None);
Assert.Equal(1, secondResult.DocumentsProcessed);
Assert.Empty(secondResult.PendingDocumentIds); // already present in cursor
Assert.Empty(secondResult.PendingMappingIds);
var refreshedRecord = await _documentStore.FindBySourceAndUriAsync(
"vndr.test",
"https://example.test/advisories/ADV-2",
CancellationToken.None);
Assert.NotNull(refreshedRecord);
Assert.Equal(documentId, refreshedRecord!.Id);
Assert.NotNull(refreshedRecord.GridFsId);
Assert.NotEqual(previousGridId, refreshedRecord.GridFsId);
var files = await filesCollection.Find(FilterDefinition<BsonDocument>.Empty).ToListAsync();
Assert.Single(files);
Assert.NotEqual(previousGridId, files[0]["_id"].AsObjectId);
}
private SourceStateSeedProcessor CreateProcessor()
=> new(
_documentStore,
_rawStorage,
_stateRepository,
_timeProvider,
NullLogger<SourceStateSeedProcessor>.Instance);
public Task InitializeAsync() => Task.CompletedTask;
public async Task DisposeAsync()
{
await _client.DropDatabaseAsync(_database.DatabaseNamespace.DatabaseName);
_runner.Dispose();
}
}

View File

@@ -4,8 +4,21 @@
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<UseConcelierTestInfra>false</UseConcelierTestInfra>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<PackageReference Include="coverlet.collector" Version="6.0.4" />
<PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" Version="9.10.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="Mongo2Go" Version="4.1.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
</ItemGroup>
</Project>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
</ItemGroup>
</Project>

View File

@@ -34,21 +34,41 @@ public sealed class AdvisoryRawServiceTests
}
[Fact]
public async Task IngestAsync_PropagatesRepositoryDuplicateResult()
{
var repository = new RecordingRepository();
var service = CreateService(repository);
var existingDocument = CreateDocument();
var expectedResult = new AdvisoryRawUpsertResult(false, CreateRecord(existingDocument));
repository.NextResult = expectedResult;
var result = await service.IngestAsync(CreateDocument(), CancellationToken.None);
Assert.False(result.Inserted);
Assert.Same(expectedResult.Record, result.Record);
}
public async Task IngestAsync_PropagatesRepositoryDuplicateResult()
{
var repository = new RecordingRepository();
var service = CreateService(repository);
var existingDocument = CreateDocument();
var expectedResult = new AdvisoryRawUpsertResult(false, CreateRecord(existingDocument));
repository.NextResult = expectedResult;
var result = await service.IngestAsync(CreateDocument(), CancellationToken.None);
Assert.False(result.Inserted);
Assert.Same(expectedResult.Record, result.Record);
}
[Fact]
public async Task IngestAsync_PreservesAliasOrderAndDuplicates()
{
var repository = new RecordingRepository();
var service = CreateService(repository);
var aliasSeries = ImmutableArray.Create("CVE-2025-0001", "CVE-2025-0001", "GHSA-xxxx", "cve-2025-0001");
var document = CreateDocument() with
{
Identifiers = new RawIdentifiers(aliasSeries, "GHSA-xxxx"),
};
repository.NextResult = new AdvisoryRawUpsertResult(true, CreateRecord(document));
await service.IngestAsync(document, CancellationToken.None);
Assert.NotNull(repository.CapturedDocument);
Assert.Equal(aliasSeries, repository.CapturedDocument!.Identifiers.Aliases);
}
private static AdvisoryRawService CreateService(RecordingRepository repository)
{
var writeGuard = new AdvisoryRawWriteGuard(new AocWriteGuard());