From d0c95cf32874e5b2fe5a7b5fac15cd52f17022b1 Mon Sep 17 00:00:00 2001 From: Vladimir Moushkov Date: Thu, 9 Oct 2025 18:59:17 +0300 Subject: [PATCH] UP --- TASKS.md | 13 + TODOS.md | 36 + docs/08_MODULE_SPECIFICATIONS.md | 9 +- docs/09_API_CLI_REFERENCE.md | 78 ++- docs/ARCHITECTURE_FEEDSER.md | 10 +- scripts/update-model-goldens.ps1 | 9 + scripts/update-model-goldens.sh | 8 + .../Commands/CommandHandlersTests.cs | 170 +++++ .../Configuration/CliBootstrapperTests.cs | 79 +++ .../Services/BackendOperationsClientTests.cs | 235 +++++++ .../StellaOps.Cli.Tests.csproj | 28 + .../Testing/TestHelpers.cs | 55 ++ src/StellaOps.Cli.Tests/UnitTest1.cs | 10 + src/StellaOps.Cli.Tests/xunit.runner.json | 3 + src/StellaOps.Cli/AGENTS.md | 27 + src/StellaOps.Cli/Commands/CommandFactory.cs | 246 +++++++ src/StellaOps.Cli/Commands/CommandHandlers.cs | 323 +++++++++ .../Configuration/CliBootstrapper.cs | 77 +++ .../Configuration/StellaOpsCliOptions.cs | 18 + src/StellaOps.Cli/Program.cs | 71 ++ src/StellaOps.Cli/Properties/AssemblyInfo.cs | 3 + .../Services/BackendOperationsClient.cs | 394 +++++++++++ .../Services/IBackendOperationsClient.cs | 16 + .../Services/IScannerExecutor.cs | 17 + .../Services/IScannerInstaller.cs | 9 + .../Services/Models/JobTriggerResult.cs | 9 + .../Services/Models/ScannerArtifactResult.cs | 3 + .../Models/Transport/JobRunResponse.cs | 27 + .../Models/Transport/JobTriggerRequest.cs | 10 + .../Models/Transport/ProblemDocument.cs | 18 + .../Services/ScannerExecutionResult.cs | 3 + src/StellaOps.Cli/Services/ScannerExecutor.cs | 274 ++++++++ .../Services/ScannerInstaller.cs | 79 +++ src/StellaOps.Cli/StellaOps.Cli.csproj | 41 ++ src/StellaOps.Cli/TASKS.md | 9 + .../Telemetry/CliActivitySource.cs | 8 + src/StellaOps.Cli/Telemetry/CliMetrics.cs | 62 ++ src/StellaOps.Cli/Telemetry/VerbosityState.cs | 8 + src/StellaOps.Cli/appsettings.json | 11 + .../StellaOps.Configuration.csproj | 18 + .../StellaOpsBootstrapOptions.cs | 64 ++ .../StellaOpsConfigurationBootstrapper.cs | 106 +++ .../StellaOpsConfigurationContext.cs | 18 + .../StellaOpsConfigurationOptions.cs | 49 ++ .../StellaOpsOptionsBinder.cs | 26 + .../JsonFeedExporter.cs | 10 +- .../TrivyDbExportPlannerTests.cs | 26 +- .../TrivyDbFeedExporterTests.cs | 5 +- .../TASKS.md | 2 +- .../TrivyDbExportPlan.cs | 8 +- .../TrivyDbExportPlanner.cs | 82 ++- .../TrivyDbFeedExporter.cs | 46 +- .../AdvisoryPrecedenceMergerTests.cs | 16 + .../AffectedPackagePrecedenceResolverTests.cs | 16 +- .../AliasGraphResolverTests.cs | 135 ++++ .../MergePrecedenceIntegrationTests.cs | 20 + .../Jobs/MergeJobKinds.cs | 6 + .../Jobs/MergeReconcileJob.cs | 43 ++ .../MergeServiceCollectionExtensions.cs | 41 ++ .../Options/AdvisoryPrecedenceDefaults.cs | 96 +++ .../Services/AdvisoryMergeService.cs | 190 ++++++ .../Services/AdvisoryPrecedenceMerger.cs | 203 +++++- .../AffectedPackagePrecedenceResolver.cs | 99 ++- .../Services/AliasGraphResolver.cs | 139 ++++ .../StellaOps.Feedser.Merge.csproj | 1 + src/StellaOps.Feedser.Merge/TASKS.md | 6 +- .../CanonicalExamplesTests.cs | 5 +- .../CanonicalJsonSerializerTests.cs | 87 +++ .../Fixtures/ghsa-semver.json | 2 + .../Fixtures/nvd-basic.json | 1 + .../Fixtures/psirt-overlay.json | 1 + .../ProvenanceDiagnosticsTests.cs | 178 +++++ .../AffectedVersionRange.cs | 6 +- .../CANONICAL_RECORDS.md | 1 + .../PROVENANCE_GUIDELINES.md | 2 + .../ProvenanceInspector.cs | 253 +++++++ .../RangePrimitives.cs | 58 ++ .../StellaOps.Feedser.Models.csproj | 3 + src/StellaOps.Feedser.Models/TASKS.md | 4 +- .../CertFr/CertFrConnectorTests.cs | 11 +- src/StellaOps.Feedser.Source.CertFr/AGENTS.md | 3 +- .../CertIn/CertInConnectorTests.cs | 16 +- .../CertIn/Fixtures/expected-advisory.json | 2 +- src/StellaOps.Feedser.Source.CertIn/AGENTS.md | 3 +- src/StellaOps.Feedser.Source.Common/AGENTS.md | 3 +- .../DebianConnectorTests.cs | 250 +++++++ .../DebianMapperTests.cs | 88 +++ .../Fixtures/debian-detail-dsa-2024-123.html | 23 + .../Fixtures/debian-detail-dsa-2024-124.html | 21 + .../Distro/Debian/Fixtures/debian-list.txt | 7 + ....Feedser.Source.Distro.Debian.Tests.csproj | 13 + .../AssemblyInfo.cs | 3 + .../Class1.cs | 29 - .../Configuration/DebianOptions.cs | 87 +++ .../DebianConnector.cs | 637 ++++++++++++++++++ .../DebianConnectorPlugin.cs | 22 + .../DebianDependencyInjectionRoutine.cs | 53 ++ .../DebianServiceCollectionExtensions.cs | 37 + .../Internal/DebianAdvisoryDto.cs | 27 + .../Internal/DebianCursor.cs | 177 +++++ .../Internal/DebianDetailMetadata.cs | 12 + .../Internal/DebianFetchCacheEntry.cs | 76 +++ .../Internal/DebianHtmlParser.cs | 326 +++++++++ .../Internal/DebianListEntry.cs | 11 + .../Internal/DebianListParser.cs | 107 +++ .../Internal/DebianMapper.cs | 266 ++++++++ .../Jobs.cs | 46 ++ ...llaOps.Feedser.Source.Distro.Debian.csproj | 3 +- .../Fixtures/rhsa-2025-0001.snapshot.json | 22 + .../Fixtures/rhsa-2025-0002.snapshot.json | 110 +++ .../Fixtures/rhsa-2025-0003.snapshot.json | 113 ++++ .../RedHat/RedHatConnectorTests.cs | 152 ++++- .../AGENTS.md | 3 +- .../Internal/RedHatMapper.cs | 44 +- .../Properties/AssemblyInfo.cs | 3 + .../RedHatConnector.cs | 2 + .../TASKS.md | 4 +- .../Distro/Suse/Fixtures/suse-changes.csv | 2 + .../Suse/Fixtures/suse-su-2025_0001-1.json | 63 ++ .../Suse/Fixtures/suse-su-2025_0002-1.json | 66 ++ ...ps.Feedser.Source.Distro.Suse.Tests.csproj | 18 + .../SuseConnectorTests.cs | 168 +++++ .../SuseCsafParserTests.cs | 52 ++ .../SuseMapperTests.cs | 52 ++ .../AssemblyInfo.cs | 3 + .../Class1.cs | 29 - .../Configuration/SuseOptions.cs | 86 +++ .../Internal/SuseAdvisoryDto.cs | 28 + .../Internal/SuseChangeRecord.cs | 5 + .../Internal/SuseChangesParser.cs | 81 +++ .../Internal/SuseCsafParser.cs | 422 ++++++++++++ .../Internal/SuseCursor.cs | 177 +++++ .../Internal/SuseFetchCacheEntry.cs | 76 +++ .../Internal/SuseMapper.cs | 313 +++++++++ .../Jobs.cs | 46 ++ ...tellaOps.Feedser.Source.Distro.Suse.csproj | 3 +- .../SuseConnector.cs | 573 ++++++++++++++++ .../SuseConnectorPlugin.cs | 20 + .../SuseDependencyInjectionRoutine.cs | 53 ++ .../SuseServiceCollectionExtensions.cs | 35 + .../Fixtures/ubuntu-notices-page0.json | 40 ++ .../Fixtures/ubuntu-notices-page1.json | 42 ++ ....Feedser.Source.Distro.Ubuntu.Tests.csproj | 18 + .../UbuntuConnectorTests.cs | 171 +++++ .../Class1.cs | 29 - .../Configuration/UbuntuOptions.cs | 69 ++ .../Internal/UbuntuCursor.cs | 177 +++++ .../Internal/UbuntuFetchCacheEntry.cs | 76 +++ .../Internal/UbuntuMapper.cs | 217 ++++++ .../Internal/UbuntuNoticeDto.cs | 25 + .../Internal/UbuntuNoticeParser.cs | 215 ++++++ .../Jobs.cs | 46 ++ ...llaOps.Feedser.Source.Distro.Ubuntu.csproj | 3 +- .../TASKS.md | 9 + .../UbuntuConnector.cs | 537 +++++++++++++++ .../UbuntuConnectorPlugin.cs | 20 + .../UbuntuDependencyInjectionRoutine.cs | 53 ++ .../UbuntuServiceCollectionExtensions.cs | 37 + .../Kaspersky/KasperskyConnectorTests.cs | 11 +- .../AGENTS.md | 3 +- .../Jvn/Fixtures/expected-advisory.json | 35 +- .../Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml | 6 + .../Jvn/JvnConnectorTests.cs | 53 +- src/StellaOps.Feedser.Source.Jvn/AGENTS.md | 3 +- .../Internal/JvnAdvisoryMapper.cs | 38 +- .../Internal/JvnDetailDto.cs | 2 + .../Internal/JvnDetailParser.cs | 40 +- .../JvnConnector.cs | 6 +- .../Schemas/vuldef_3.2.xsd | 2 + src/StellaOps.Feedser.Source.Jvn/TASKS.md | 2 +- .../Nvd/NvdConnectorHarnessTests.cs | 16 +- .../Nvd/NvdConnectorTests.cs | 48 +- src/StellaOps.Feedser.Source.Nvd/AGENTS.md | 3 +- .../Internal/NvdMapper.cs | 211 +++++- .../Fixtures/osv-npm.snapshot.json | 115 ++++ .../Fixtures/osv-pypi.snapshot.json | 115 ++++ .../Osv/OsvMapperTests.cs | 6 + .../Osv/OsvSnapshotTests.cs | 141 ++++ .../StellaOps.Feedser.Source.Osv.Tests.csproj | 5 + src/StellaOps.Feedser.Source.Osv/AGENTS.md | 3 +- .../Internal/OsvMapper.cs | 23 +- .../OsvConnector.cs | 3 + .../StellaOps.Feedser.Source.Osv.csproj | 3 + src/StellaOps.Feedser.Source.Osv/TASKS.md | 6 +- .../Adobe/AdobeConnectorFetchTests.cs | 96 ++- .../Fixtures/adobe-advisories.snapshot.json | 354 +++++++++- .../Fixtures/adobe-detail-apsb25-85.html | 64 +- .../Fixtures/adobe-detail-apsb25-87.html | 44 +- ...Ops.Feedser.Source.Vndr.Adobe.Tests.csproj | 4 +- .../AGENTS.md | 3 +- .../AdobeConnector.cs | 247 ++++++- .../Internal/AdobeBulletinDto.cs | 66 +- .../Internal/AdobeDetailParser.cs | 326 ++++++++- .../Schemas/adobe-bulletin.schema.json | 36 +- .../Chromium/ChromiumConnectorTests.cs | 18 +- .../Chromium/ChromiumMapperTests.cs | 2 +- .../Fixtures/chromium-advisory.snapshot.json | 2 +- .../AGENTS.md | 3 +- .../ChromiumConnector.cs | 2 + .../Internal/ChromiumMapper.cs | 56 +- .../TASKS.md | 1 + .../Fixtures/oracle-advisories.snapshot.json | 395 ++++++++++- .../oracle-calendar-cpuapr2024-single.html | 7 + .../Fixtures/oracle-calendar-cpuapr2024.html | 8 + .../Fixtures/oracle-detail-cpuapr2024-01.html | 103 ++- .../Fixtures/oracle-detail-cpuapr2024-02.html | 101 ++- .../Fixtures/oracle-detail-invalid.html | 4 + .../Oracle/OracleConnectorTests.cs | 205 +++++- ...ps.Feedser.Source.Vndr.Oracle.Tests.csproj | 1 + .../AGENTS.md | 3 +- .../Configuration/OracleOptions.cs | 11 +- .../Internal/OracleAffectedEntry.cs | 10 + .../Internal/OracleCalendarFetcher.cs | 92 +++ .../Internal/OracleCursor.cs | 149 +++- .../Internal/OracleDto.cs | 5 +- .../Internal/OracleDtoValidator.cs | 276 ++++++++ .../Internal/OracleMapper.cs | 380 ++++++++++- .../Internal/OracleParser.cs | 412 ++++++++++- .../Internal/OraclePatchDocument.cs | 8 + .../Jobs.cs | 0 .../OracleConnector.cs | 75 ++- .../OracleDependencyInjectionRoutine.cs | 0 .../OracleServiceCollectionExtensions.cs | 5 + .../Properties/AssemblyInfo.cs | 3 + .../TASKS.md | 19 +- ...ps.Feedser.Source.Vndr.Vmware.Tests.csproj | 5 + .../Fixtures/vmware-advisories.snapshot.json | 258 +++++++ .../vmware-detail-vmsa-2024-0001.json | 33 + .../vmware-detail-vmsa-2024-0002.json | 27 + .../vmware-detail-vmsa-2024-0003.json | 23 + .../Vmware/Fixtures/vmware-index-initial.json | 12 + .../Vmware/Fixtures/vmware-index-second.json | 17 + .../Vmware/VmwareConnectorTests.cs | 266 ++++++++ .../Vmware/VmwareMapperTests.cs | 5 +- .../AGENTS.md | 3 +- .../Internal/VmwareCursor.cs | 70 +- .../Internal/VmwareFetchCacheEntry.cs | 88 +++ .../Internal/VmwareMapper.cs | 99 ++- .../Properties/AssemblyInfo.cs | 3 + .../TASKS.md | 19 +- .../VmwareConnector.cs | 88 ++- .../VmwareDiagnostics.cs | 67 ++ .../VmwareServiceCollectionExtensions.cs | 2 + .../AdvisoryStorePerformanceTests.cs | 4 +- .../AdvisoryStoreTests.cs | 114 +++- .../AliasStoreTests.cs | 60 ++ .../ExportStateManagerTests.cs | 9 + .../ExportStateStoreTests.cs | 6 +- .../MongoJobStoreTests.cs | 21 +- .../RawDocumentRetentionServiceTests.cs | 9 +- .../Advisories/AdvisoryStore.cs | 153 ++++- .../Aliases/AliasDocument.cs | 38 ++ .../Aliases/AliasStore.cs | 157 +++++ .../Aliases/AliasStoreConstants.cs | 7 + .../Aliases/AliasStoreMetrics.cs | 22 + .../Aliases/IAliasStore.cs | 27 + .../ChangeHistory/ChangeHistoryDocument.cs | 4 +- .../ChangeHistoryDocumentExtensions.cs | 8 +- .../Documents/DocumentDocument.cs | 7 +- .../Documents/DocumentStore.cs | 6 +- .../Dtos/DtoDocument.cs | 13 +- .../Dtos/DtoStore.cs | 6 +- .../Exporting/ExportStateDocument.cs | 29 +- .../Exporting/ExportStateManager.cs | 11 +- .../Exporting/ExportStateRecord.cs | 5 +- .../JobRunDocument.cs | 8 +- .../MergeEvents/MergeEventDocument.cs | 16 +- .../MongoBootstrapper.cs | 22 +- .../MongoJobStore.cs | 8 +- .../ServiceCollectionExtensions.cs | 2 + src/StellaOps.Feedser.Storage.Mongo/TASKS.md | 1 + .../MongoIntegrationFixture.cs | 6 +- src/StellaOps.Feedser.WebService/AGENTS.md | 4 +- .../Extensions/JobRegistrationExtensions.cs | 4 +- src/StellaOps.Feedser.WebService/Program.cs | 74 +- .../StellaOps.Feedser.WebService.csproj | 2 + src/StellaOps.Feedser.sln | 56 ++ 277 files changed, 17449 insertions(+), 595 deletions(-) create mode 100644 TASKS.md create mode 100644 TODOS.md create mode 100644 scripts/update-model-goldens.ps1 create mode 100644 scripts/update-model-goldens.sh create mode 100644 src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs create mode 100644 src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs create mode 100644 src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs create mode 100644 src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj create mode 100644 src/StellaOps.Cli.Tests/Testing/TestHelpers.cs create mode 100644 src/StellaOps.Cli.Tests/UnitTest1.cs create mode 100644 src/StellaOps.Cli.Tests/xunit.runner.json create mode 100644 src/StellaOps.Cli/AGENTS.md create mode 100644 src/StellaOps.Cli/Commands/CommandFactory.cs create mode 100644 src/StellaOps.Cli/Commands/CommandHandlers.cs create mode 100644 src/StellaOps.Cli/Configuration/CliBootstrapper.cs create mode 100644 src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs create mode 100644 src/StellaOps.Cli/Program.cs create mode 100644 src/StellaOps.Cli/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Cli/Services/BackendOperationsClient.cs create mode 100644 src/StellaOps.Cli/Services/IBackendOperationsClient.cs create mode 100644 src/StellaOps.Cli/Services/IScannerExecutor.cs create mode 100644 src/StellaOps.Cli/Services/IScannerInstaller.cs create mode 100644 src/StellaOps.Cli/Services/Models/JobTriggerResult.cs create mode 100644 src/StellaOps.Cli/Services/Models/ScannerArtifactResult.cs create mode 100644 src/StellaOps.Cli/Services/Models/Transport/JobRunResponse.cs create mode 100644 src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs create mode 100644 src/StellaOps.Cli/Services/Models/Transport/ProblemDocument.cs create mode 100644 src/StellaOps.Cli/Services/ScannerExecutionResult.cs create mode 100644 src/StellaOps.Cli/Services/ScannerExecutor.cs create mode 100644 src/StellaOps.Cli/Services/ScannerInstaller.cs create mode 100644 src/StellaOps.Cli/StellaOps.Cli.csproj create mode 100644 src/StellaOps.Cli/TASKS.md create mode 100644 src/StellaOps.Cli/Telemetry/CliActivitySource.cs create mode 100644 src/StellaOps.Cli/Telemetry/CliMetrics.cs create mode 100644 src/StellaOps.Cli/Telemetry/VerbosityState.cs create mode 100644 src/StellaOps.Cli/appsettings.json create mode 100644 src/StellaOps.Configuration/StellaOps.Configuration.csproj create mode 100644 src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs create mode 100644 src/StellaOps.Configuration/StellaOpsConfigurationBootstrapper.cs create mode 100644 src/StellaOps.Configuration/StellaOpsConfigurationContext.cs create mode 100644 src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs create mode 100644 src/StellaOps.Configuration/StellaOpsOptionsBinder.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs create mode 100644 src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs create mode 100644 src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs create mode 100644 src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs create mode 100644 src/StellaOps.Feedser.Models/ProvenanceInspector.cs create mode 100644 src/StellaOps.Feedser.Models/RangePrimitives.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs delete mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs delete mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs delete mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs rename src/{ => StellaOps.Feedser.Source.Vndr.Oracle}/Jobs.cs (100%) rename src/{ => StellaOps.Feedser.Source.Vndr.Oracle}/OracleDependencyInjectionRoutine.cs (100%) create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs diff --git a/TASKS.md b/TASKS.md new file mode 100644 index 00000000..8ced7f58 --- /dev/null +++ b/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Merge identity graph & alias store|BE-Merge|Models, Storage.Mongo|**DONE** – alias store/resolver, component builder, reconcile job, persistence + diagnostics endpoint landed.| +|OSV alias consolidation & per-ecosystem snapshots|BE-Conn-OSV, QA|Merge, Testing|DONE – alias graph handles GHSA/CVE records and deterministic snapshots exist across ecosystems.| +|Oracle PSIRT pipeline completion|BE-Conn-Oracle|Source.Common, Core|**DONE** – Oracle mapper now emits CVE aliases, vendor affected packages, patch references, and resume/backfill flow is covered by integration tests.| +|VMware connector observability & resume coverage|BE-Conn-VMware, QA|Source.Common, Storage.Mongo|**DONE** – VMware diagnostics emit fetch/parse/map metrics, fetch dedupe uses hash cache, and integration test covers snapshot plus resume path.| +|Model provenance & range backlog|BE-Merge|Models|**DOING** – VMware/Oracle/Chromium, NVD, Debian, SUSE, Ubuntu, and Adobe emit RangePrimitives (Debian EVR + SUSE NEVRA + Ubuntu EVR telemetry online; Adobe now reports `adobe.track/platform/priority/availability` telemetry with fixed-status provenance). Remaining connectors (Apple, etc.) still need structured primitives/EVR coverage.| +|Trivy DB exporter delta strategy|BE-Export|Exporters|**TODO** – finish `ExportStateManager` delta reset and design incremental layer reuse for unchanged trees.| +|Red Hat fixture validation sweep|QA|Source.Distro.RedHat|**DOING** – finalize RHSA fixture regeneration once connector regression fixes land.| +|JVN VULDEF schema update|BE-Conn-JVN, QA|Source.Jvn|**DONE** – schema patched (vendor/product attrs, impact entries, err codes), parser tightened, fixtures/tests refreshed.| +|Build/test sweeps|QA|All modules|**DOING** – targeted suites green (Models, VMware, Oracle, Chromium, JVN, Cert-In). Full solution run still fails due to `StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests` exceeding perf budget; rerun once budget or test adjusted.| +|OSV vs GHSA parity checks|QA, BE-Merge|Merge|**TODO** – design diff detection between OSV and GHSA feeds to surface inconsistencies.| diff --git a/TODOS.md b/TODOS.md new file mode 100644 index 00000000..496dfadd --- /dev/null +++ b/TODOS.md @@ -0,0 +1,36 @@ +# Pending Task Backlog + +> Last updated: 2025-10-09 (UTC) + +## Common + +- **Build/test sweeps (QA – DOING)** + Full solution runs still fail the `StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests` budget. We need either to optimise the hot paths in `AdvisoryStore` for large advisory payloads or relax the perf thresholds with new baseline data. Once the bottleneck is addressed, rerun the full suite and capture metrics for the release checklist. + +- **OSV vs GHSA parity checks (QA & BE-Merge – TODO)** + Design and implement a diff detector comparing OSV advisories against GHSA records. The deliverable should flag mismatched aliases, missing affected ranges, or divergent severities, surface actionable telemetry/alerts, and include regression tests with canned OSV+GHSA fixtures. + +## Prerequisites + +- **Range primitives for SemVer/EVR/NEVRA metadata (BE-Merge – DOING)** + The core model supports range primitives, but several connectors (notably Apple, remaining vendor feeds, and older distro paths) still emit raw strings. We must extend those mappers to populate the structured envelopes (SemVer/EVR/NEVRA plus vendor extensions) and add fixture coverage so merge/export layers see consistent telemetry. + +- **Provenance envelope field masks (BE-Merge – DOING)** + Provenance needs richer categorisation (component category, severity bands, resume counters) and better dedupe metrics. Update the provenance model, extend diagnostics to emit the new tags, and refresh dashboards/tests to ensure determinism once additional metadata flows through. + +## Implementations + +- **Model provenance & range backlog (BE-Merge – DOING)** + With Adobe/Ubuntu now emitting range primitives, focus on the remaining connectors (e.g., Apple, smaller vendor PSIRTs). Update their pipelines, regenerate goldens, and confirm `feedser.range.primitives` metrics reflect the added telemetry. The task closes when every high-priority source produces structured ranges with provenance. + +- **Trivy DB exporter delta strategy (BE-Export – TODO)** + Finalise the delta-reset story in `ExportStateManager`: define when to invalidate baselines, how to reuse unchanged layers, and document operator workflows. Implement planner logic for layer reuse, update exporter tests, and exercise a delta→full→delta sequence. + +- **Red Hat fixture validation sweep (QA – DOING)** + Regenerate RHSA fixtures with the latest connector output and make sure the regenerated snapshots align once the outstanding connector tweaks land. Blockers: connector regression fixes still in-flight; revisit once those merges stabilise to avoid churn. + +- **Plan incremental/delta exports (BE-Export – DOING)** + `TrivyDbExportPlanner` now captures changed files but does not yet reuse existing OCI layers. Extend the planner to build per-file manifests, teach the writer to skip untouched layers, and add delta-cycle tests covering file removals, additions, and checksum changes. + +- **Scan execution & result upload workflow (DevEx/CLI & Ops Integrator – DOING)** + `stella scan run`/`stella scan upload` need completion: support the remaining executor backends (dotnet/self-hosted/docker), capture structured run metadata, implement retry/backoff on uploads, and add integration tests exercising happy-path and failure retries. Update CLI docs once the workflow is stable. diff --git a/docs/08_MODULE_SPECIFICATIONS.md b/docs/08_MODULE_SPECIFICATIONS.md index 53497ddd..a77ff817 100755 --- a/docs/08_MODULE_SPECIFICATIONS.md +++ b/docs/08_MODULE_SPECIFICATIONS.md @@ -154,9 +154,12 @@ Each connector ships fixtures/tests under the matching `*.Tests` project. * **Connector/exporter packages** – each source/exporter can ship as a plug-in assembly with its own options and HttpClient configuration, keeping the core image minimal. -* **Stella CLI (agent)** – triggers feed-related jobs (`stella db fetch/merge/export`) - and consumes the exported JSON/Trivy DB artefacts, aligning with the SBOM-first - workflow described in `AGENTS.md`. +* **StellaOps CLI (agent)** – new `StellaOps.Cli` module that exposes + `scanner`, `scan`, and `db` verbs (via System.CommandLine 2.0) to download + scanner container bundles, install them locally, execute scans against target + directories, automatically upload results, and trigger Feedser jobs (`db + fetch/merge/export`) aligned with the SBOM-first workflow described in + `AGENTS.md`. * **Offline Kit** – bundles Feedser plug-ins, JSON tree, Trivy DB, and export manifests so air-gapped sites can load the latest vulnerability data without outbound connectivity. diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md index 7f456d02..c7ec500a 100755 --- a/docs/09_API_CLI_REFERENCE.md +++ b/docs/09_API_CLI_REFERENCE.md @@ -183,21 +183,79 @@ Validation errors come back as: --- -### 2.4 Attestation (Planned – Q1‑2026) - -``` -POST /attest -``` +### 2.4 Attestation (Planned – Q1‑2026) + +``` +POST /attest +``` | Param | Purpose | | ----------- | ------------------------------------- | | body (JSON) | SLSA v1.0 provenance doc | | | Signed + stored in local Rekor mirror | - -Returns `202 Accepted` and `Location: /attest/{id}` for async verify. - ---- - + +Returns `202 Accepted` and `Location: /attest/{id}` for async verify. + +--- + +## 3 StellaOps CLI (`stellaops-cli`) + +The new CLI is built on **System.CommandLine 2.0.0‑beta5** and mirrors the Feedser backend REST API. +Configuration follows the same precedence chain everywhere: + +1. Environment variables (e.g. `API_KEY`, `STELLAOPS_BACKEND_URL`, `StellaOps:ApiKey`) +2. `appsettings.json` → `appsettings.local.json` +3. `appsettings.yaml` → `appsettings.local.yaml` +4. Defaults (`ApiKey = ""`, `BackendUrl = ""`, cache folders under the current working directory) + +| Command | Purpose | Key Flags / Arguments | Notes | +|---------|---------|-----------------------|-------| +| `stellaops-cli scanner download` | Fetch and install scanner container | `--channel ` (default `stable`)
`--output `
`--overwrite`
`--no-install` | Saves artefact under `ScannerCacheDirectory`, verifies digest/signature, and executes `docker load` unless `--no-install` is supplied. | +| `stellaops-cli scan run` | Execute scanner container against a directory (auto-upload) | `--target ` (required)
`--runner ` (default from config)
`--entry `
`[scanner-args...]` | Runs the scanner, writes results into `ResultsDirectory`, and automatically uploads the artefact when the exit code is `0`. | +| `stellaops-cli scan upload` | Re-upload existing scan artefact | `--file ` | Useful for retries when automatic upload fails or when operating offline. | +| `stellaops-cli db fetch` | Trigger connector jobs | `--source ` (e.g. `redhat`, `osv`)
`--stage ` (default `fetch`)
`--mode ` | Translates to `POST /jobs/source:{source}:{stage}` with `trigger=cli` | +| `stellaops-cli db merge` | Run canonical merge reconcile | — | Calls `POST /jobs/merge:reconcile`; exit code `0` on acceptance, `1` on failures/conflicts | +| `stellaops-cli db export` | Kick JSON / Trivy exports | `--format ` (default `json`)
`--delta` | Sets `{ delta = true }` parameter when requested | +| `stellaops-cli config show` | Display resolved configuration | — | Masks secret values; helpful for air‑gapped installs | + +**Logging & exit codes** + +- Structured logging via `Microsoft.Extensions.Logging` with single-line console output (timestamps in UTC). +- `--verbose / -v` raises log level to `Debug`. +- Command exit codes bubble up: backend conflict → `1`, cancelled via `CTRL+C` → `130`, scanner exit codes propagate as-is. + +**Artifact validation** + +- Downloads are verified against the `X-StellaOps-Digest` header (SHA-256). When `StellaOps:ScannerSignaturePublicKeyPath` points to a PEM-encoded RSA key, the optional `X-StellaOps-Signature` header is validated as well. +- Metadata for each bundle is written alongside the artefact (`*.metadata.json`) with digest, signature, source URL, and timestamps. +- Retry behaviour is controlled via `StellaOps:ScannerDownloadAttempts` (default **3** with exponential backoff). +- Successful `scan run` executions create timestamped JSON artefacts inside `ResultsDirectory`; these are posted back to Feedser automatically. + +**Authentication** + +- API key is sent as `Authorization: Bearer ` automatically when configured. +- Anonymous operation (empty key) is permitted for offline use cases but backend calls will fail with 401 unless the Feedser instance allows guest access. + +**Configuration file template** + +```jsonc +{ + "StellaOps": { + "ApiKey": "your-api-token", + "BackendUrl": "https://feedser.example.org", + "ScannerCacheDirectory": "scanners", + "ResultsDirectory": "results", + "DefaultRunner": "docker", + "ScannerSignaturePublicKeyPath": "", + "ScannerDownloadAttempts": 3 + } +} +``` + +Drop `appsettings.local.json` or `.yaml` beside the binary to override per environment. + +--- + ### 2.5 Misc Endpoints | Path | Method | Description | diff --git a/docs/ARCHITECTURE_FEEDSER.md b/docs/ARCHITECTURE_FEEDSER.md index f1df1515..c495be87 100644 --- a/docs/ARCHITECTURE_FEEDSER.md +++ b/docs/ARCHITECTURE_FEEDSER.md @@ -47,14 +47,14 @@ StellaOps.Feedser.Source.Ru.Nkcki/ # PDF/HTML bulletins → structured StellaOps.Feedser.Source.Vndr.Msrc/ StellaOps.Feedser.Source.Vndr.Cisco/ StellaOps.Feedser.Source.Vndr.Oracle/ -StellaOps.Feedser.Source.Vndr.Adobe/ +StellaOps.Feedser.Source.Vndr.Adobe/ # APSB ingest; emits vendor RangePrimitives with adobe.track/platform/priority telemetry + fixed-status provenance. StellaOps.Feedser.Source.Vndr.Apple/ StellaOps.Feedser.Source.Vndr.Chromium/ StellaOps.Feedser.Source.Vndr.Vmware/ StellaOps.Feedser.Source.Distro.RedHat/ -StellaOps.Feedser.Source.Distro.Ubuntu/ -StellaOps.Feedser.Source.Distro.Debian/ -StellaOps.Feedser.Source.Distro.Suse/ +StellaOps.Feedser.Source.Distro.Debian/ # Fetches DSA list + detail HTML, emits EVR RangePrimitives with per-release provenance and telemetry. +StellaOps.Feedser.Source.Distro.Ubuntu/ # Ubuntu Security Notices connector (JSON index → EVR ranges with ubuntu.pocket telemetry). +StellaOps.Feedser.Source.Distro.Suse/ # CSAF fetch pipeline emitting NEVRA RangePrimitives with suse.status vendor telemetry. StellaOps.Feedser.Source.Ics.Cisa/ StellaOps.Feedser.Source.Ics.Kaspersky/ StellaOps.Feedser.Normalization/ # Canonical mappers, validators, version-range normalization @@ -169,7 +169,7 @@ public interface IFeedConnector { ## 8) Observability * Serilog structured logging with enrichment fields (`source`, `uri`, `stage`, `durationMs`). -* OpenTelemetry traces around fetch/parse/map/export; metrics for rate limit hits, schema failures, dedupe ratios, package size. +* OpenTelemetry traces around fetch/parse/map/export; metrics for rate limit hits, schema failures, dedupe ratios, package size. Connector HTTP metrics are emitted via the shared `feedser.source.http.*` instruments tagged with `feedser.source=` so per-source dashboards slice on that label instead of bespoke metric names. * Prometheus scraping endpoint served by WebService. --- diff --git a/scripts/update-model-goldens.ps1 b/scripts/update-model-goldens.ps1 new file mode 100644 index 00000000..a726ace4 --- /dev/null +++ b/scripts/update-model-goldens.ps1 @@ -0,0 +1,9 @@ +Param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]] $RestArgs +) + +$Root = Split-Path -Parent $PSScriptRoot +$env:UPDATE_GOLDENS = "1" + +dotnet test (Join-Path $Root "src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj") @RestArgs diff --git a/scripts/update-model-goldens.sh b/scripts/update-model-goldens.sh new file mode 100644 index 00000000..cb1aa8da --- /dev/null +++ b/scripts/update-model-goldens.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +export UPDATE_GOLDENS=1 + +dotnet test "$ROOT_DIR/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj" "$@" diff --git a/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs b/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs new file mode 100644 index 00000000..be1e24aa --- /dev/null +++ b/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Cli.Commands; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Telemetry; +using StellaOps.Cli.Tests.Testing; + +namespace StellaOps.Cli.Tests.Commands; + +public sealed class CommandHandlersTests +{ + [Fact] + public async Task HandleExportJobAsync_SetsExitCodeZeroOnSuccess() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", "/jobs/export:json/1", null)); + var provider = BuildServiceProvider(backend); + + await CommandHandlers.HandleExportJobAsync(provider, "json", delta: false, verbose: false, CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + Assert.Equal("export:json", backend.LastJobKind); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleMergeJobAsync_SetsExitCodeOnFailure() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(false, "Job already running", null, null)); + var provider = BuildServiceProvider(backend); + + await CommandHandlers.HandleMergeJobAsync(provider, verbose: false, CancellationToken.None); + + Assert.Equal(1, Environment.ExitCode); + Assert.Equal("merge:reconcile", backend.LastJobKind); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleScannerRunAsync_AutomaticallyUploadsResults() + { + using var tempDir = new TempDirectory(); + var resultsFile = Path.Combine(tempDir.Path, "results", "scan.json"); + var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", null, null)); + var executor = new StubExecutor(new ScannerExecutionResult(0, resultsFile)); + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results") + }; + + var provider = BuildServiceProvider(backend, executor, new StubInstaller(), options); + + Directory.CreateDirectory(Path.Combine(tempDir.Path, "target")); + + var original = Environment.ExitCode; + try + { + await CommandHandlers.HandleScannerRunAsync( + provider, + runner: "docker", + entry: "scanner-image", + targetDirectory: Path.Combine(tempDir.Path, "target"), + arguments: Array.Empty(), + verbose: false, + cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + Assert.Equal(resultsFile, backend.LastUploadPath); + } + finally + { + Environment.ExitCode = original; + } + } + + private static IServiceProvider BuildServiceProvider( + IBackendOperationsClient backend, + IScannerExecutor? executor = null, + IScannerInstaller? installer = null, + StellaOpsCliOptions? options = null) + { + var services = new ServiceCollection(); + services.AddSingleton(backend); + services.AddSingleton(_ => LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug))); + services.AddSingleton(new VerbosityState()); + services.AddSingleton(options ?? new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(Path.GetTempPath(), $"stellaops-cli-results-{Guid.NewGuid():N}") + }); + services.AddSingleton(executor ?? new StubExecutor(new ScannerExecutionResult(0, Path.GetTempFileName()))); + services.AddSingleton(installer ?? new StubInstaller()); + + return services.BuildServiceProvider(); + } + + private sealed class StubBackendClient : IBackendOperationsClient + { + private readonly JobTriggerResult _result; + + public StubBackendClient(JobTriggerResult result) + { + _result = result; + } + + public string? LastJobKind { get; private set; } + public string? LastUploadPath { get; private set; } + + public Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) + { + LastUploadPath = filePath; + return Task.CompletedTask; + } + + public Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken) + { + LastJobKind = jobKind; + return Task.FromResult(_result); + } + } + + private sealed class StubExecutor : IScannerExecutor + { + private readonly ScannerExecutionResult _result; + + public StubExecutor(ScannerExecutionResult result) + { + _result = result; + } + + public Task RunAsync(string runner, string entry, string targetDirectory, string resultsDirectory, IReadOnlyList arguments, bool verbose, CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(_result.ResultsPath)!); + if (!File.Exists(_result.ResultsPath)) + { + File.WriteAllText(_result.ResultsPath, "{}"); + } + + return Task.FromResult(_result); + } + } + + private sealed class StubInstaller : IScannerInstaller + { + public Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken) + => Task.CompletedTask; + } +} diff --git a/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs b/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs new file mode 100644 index 00000000..e4e7d113 --- /dev/null +++ b/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs @@ -0,0 +1,79 @@ +using System; +using System.IO; +using System.Text.Json; +using StellaOps.Cli.Configuration; +using Xunit; + +namespace StellaOps.Cli.Tests.Configuration; + +public sealed class CliBootstrapperTests : IDisposable +{ + private readonly string _originalDirectory = Directory.GetCurrentDirectory(); + private readonly string _tempDirectory = Path.Combine(Path.GetTempPath(), $"stellaops-cli-tests-{Guid.NewGuid():N}"); + + public CliBootstrapperTests() + { + Directory.CreateDirectory(_tempDirectory); + Directory.SetCurrentDirectory(_tempDirectory); + } + + [Fact] + public void Build_UsesEnvironmentVariablesWhenPresent() + { + Environment.SetEnvironmentVariable("API_KEY", "env-key"); + Environment.SetEnvironmentVariable("STELLAOPS_BACKEND_URL", "https://env-backend.example"); + + try + { + var (options, _) = CliBootstrapper.Build(Array.Empty()); + + Assert.Equal("env-key", options.ApiKey); + Assert.Equal("https://env-backend.example", options.BackendUrl); + } + finally + { + Environment.SetEnvironmentVariable("API_KEY", null); + Environment.SetEnvironmentVariable("STELLAOPS_BACKEND_URL", null); + } + } + + [Fact] + public void Build_FallsBackToAppSettings() + { + WriteAppSettings(new + { + StellaOps = new + { + ApiKey = "file-key", + BackendUrl = "https://file-backend.example" + } + }); + + var (options, _) = CliBootstrapper.Build(Array.Empty()); + + Assert.Equal("file-key", options.ApiKey); + Assert.Equal("https://file-backend.example", options.BackendUrl); + } + + public void Dispose() + { + Directory.SetCurrentDirectory(_originalDirectory); + if (Directory.Exists(_tempDirectory)) + { + try + { + Directory.Delete(_tempDirectory, recursive: true); + } + catch + { + // Ignored. + } + } + } + + private static void WriteAppSettings(T payload) + { + var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText("appsettings.json", json); + } +} diff --git a/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs b/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs new file mode 100644 index 00000000..891582fa --- /dev/null +++ b/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs @@ -0,0 +1,235 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Services.Models.Transport; +using StellaOps.Cli.Tests.Testing; + +namespace StellaOps.Cli.Tests.Services; + +public sealed class BackendOperationsClientTests +{ + [Fact] + public async Task DownloadScannerAsync_VerifiesDigestAndWritesMetadata() + { + using var temp = new TempDirectory(); + + var contentBytes = Encoding.UTF8.GetBytes("scanner-blob"); + var digestHex = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant(); + + var handler = new StubHttpMessageHandler((request, _) => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(contentBytes), + RequestMessage = request + }; + + response.Headers.Add("X-StellaOps-Digest", $"sha256:{digestHex}"); + response.Content.Headers.LastModified = DateTimeOffset.UtcNow; + response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream"); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + ScannerCacheDirectory = temp.Path, + ScannerDownloadAttempts = 1 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); + var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: true, CancellationToken.None); + + Assert.False(result.FromCache); + Assert.True(File.Exists(targetPath)); + + var metadataPath = targetPath + ".metadata.json"; + Assert.True(File.Exists(metadataPath)); + + using var document = JsonDocument.Parse(File.ReadAllText(metadataPath)); + Assert.Equal($"sha256:{digestHex}", document.RootElement.GetProperty("digest").GetString()); + Assert.Equal("stable", document.RootElement.GetProperty("channel").GetString()); + } + + [Fact] + public async Task DownloadScannerAsync_ThrowsOnDigestMismatch() + { + using var temp = new TempDirectory(); + + var contentBytes = Encoding.UTF8.GetBytes("scanner-data"); + var handler = new StubHttpMessageHandler((request, _) => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(contentBytes), + RequestMessage = request + }; + response.Headers.Add("X-StellaOps-Digest", "sha256:deadbeef"); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + ScannerCacheDirectory = temp.Path, + ScannerDownloadAttempts = 1 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); + + await Assert.ThrowsAsync(() => client.DownloadScannerAsync("stable", targetPath, overwrite: true, verbose: false, CancellationToken.None)); + Assert.False(File.Exists(targetPath)); + } + + [Fact] + public async Task DownloadScannerAsync_RetriesOnFailure() + { + using var temp = new TempDirectory(); + + var successBytes = Encoding.UTF8.GetBytes("success"); + var digestHex = Convert.ToHexString(SHA256.HashData(successBytes)).ToLowerInvariant(); + var attempts = 0; + + var handler = new StubHttpMessageHandler( + (request, _) => + { + attempts++; + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + RequestMessage = request, + Content = new StringContent("error") + }; + }, + (request, _) => + { + attempts++; + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new ByteArrayContent(successBytes) + }; + response.Headers.Add("X-StellaOps-Digest", $"sha256:{digestHex}"); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://feedser.example", + ScannerCacheDirectory = temp.Path, + ScannerDownloadAttempts = 3 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var targetPath = Path.Combine(temp.Path, "scanner.tar.gz"); + var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: false, CancellationToken.None); + + Assert.Equal(2, attempts); + Assert.False(result.FromCache); + Assert.True(File.Exists(targetPath)); + } + + [Fact] + public async Task TriggerJobAsync_ReturnsAcceptedResult() + { + var handler = new StubHttpMessageHandler((request, _) => + { + var response = new HttpResponseMessage(HttpStatusCode.Accepted) + { + RequestMessage = request, + Content = JsonContent.Create(new JobRunResponse + { + RunId = Guid.NewGuid(), + Status = "queued", + Kind = "export:json", + Trigger = "cli", + CreatedAt = DateTimeOffset.UtcNow + }) + }; + response.Headers.Location = new Uri("/jobs/export:json/runs/123", UriKind.Relative); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions { BackendUrl = "https://feedser.example" }; + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var result = await client.TriggerJobAsync("export:json", new Dictionary(), CancellationToken.None); + + Assert.True(result.Success); + Assert.Equal("Accepted", result.Message); + Assert.Equal("/jobs/export:json/runs/123", result.Location); + } + + [Fact] + public async Task TriggerJobAsync_ReturnsFailureMessage() + { + var handler = new StubHttpMessageHandler((request, _) => + { + var problem = new + { + title = "Job already running", + detail = "export job active" + }; + + var response = new HttpResponseMessage(HttpStatusCode.Conflict) + { + RequestMessage = request, + Content = JsonContent.Create(problem) + }; + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://feedser.example") + }; + + var options = new StellaOpsCliOptions { BackendUrl = "https://feedser.example" }; + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var result = await client.TriggerJobAsync("export:json", new Dictionary(), CancellationToken.None); + + Assert.False(result.Success); + Assert.Contains("Job already running", result.Message); + } +} diff --git a/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj b/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj new file mode 100644 index 00000000..5ce89715 --- /dev/null +++ b/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj @@ -0,0 +1,28 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs b/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs new file mode 100644 index 00000000..0b412b25 --- /dev/null +++ b/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Cli.Tests.Testing; + +internal sealed class TempDirectory : IDisposable +{ + public TempDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-cli-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + catch + { + // ignored + } + } +} + +internal sealed class StubHttpMessageHandler : HttpMessageHandler +{ + private readonly Queue> _responses; + + public StubHttpMessageHandler(params Func[] handlers) + { + if (handlers is null || handlers.Length == 0) + { + throw new ArgumentException("At least one handler must be provided.", nameof(handlers)); + } + + _responses = new Queue>(handlers); + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var factory = _responses.Count > 1 ? _responses.Dequeue() : _responses.Peek(); + return Task.FromResult(factory(request, cancellationToken)); + } +} diff --git a/src/StellaOps.Cli.Tests/UnitTest1.cs b/src/StellaOps.Cli.Tests/UnitTest1.cs new file mode 100644 index 00000000..e584b0dc --- /dev/null +++ b/src/StellaOps.Cli.Tests/UnitTest1.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Cli.Tests; + +public class UnitTest1 +{ + [Fact] + public void Test1() + { + + } +} diff --git a/src/StellaOps.Cli.Tests/xunit.runner.json b/src/StellaOps.Cli.Tests/xunit.runner.json new file mode 100644 index 00000000..86c7ea05 --- /dev/null +++ b/src/StellaOps.Cli.Tests/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json" +} diff --git a/src/StellaOps.Cli/AGENTS.md b/src/StellaOps.Cli/AGENTS.md new file mode 100644 index 00000000..1baf8bcd --- /dev/null +++ b/src/StellaOps.Cli/AGENTS.md @@ -0,0 +1,27 @@ +# StellaOps.Cli — Agent Brief + +## Mission +- Deliver an offline-capable command-line interface that drives StellaOps back-end operations: scanner distribution, scan execution, result uploads, and Feedser database lifecycle calls (init/resume/export). +- Honour StellaOps principles of determinism, observability, and offline-first behaviour while providing a polished operator experience. + +## Role Charter +| Role | Mandate | Collaboration | +| --- | --- | --- | +| **DevEx/CLI** | Own CLI UX, command routing, and configuration model. Ensure commands work with empty/default config and document overrides. | Coordinate with Backend/WebService for API contracts and with Docs for operator workflows. | +| **Ops Integrator** | Maintain integration paths for shell/dotnet/docker tooling. Validate that air-gapped runners can bootstrap required binaries. | Work with Feedser/Agent teams to mirror packaging and signing requirements. | +| **QA** | Provide command-level fixtures, golden outputs, and regression coverage (unit & smoke). Ensure commands respect cancellation and deterministic logging. | Partner with QA guild for shared harnesses and test data. | + +## Working Agreements +- Configuration is centralised in `StellaOps.Configuration`; always consume the bootstrapper instead of hand rolling builders. Env vars (`API_KEY`, `STELLAOPS_BACKEND_URL`, `StellaOps:*`) override JSON/YAML and default to empty values. +- Command verbs (`scanner`, `scan`, `db`, `config`) are wired through System.CommandLine 2.0; keep handlers composable, cancellation-aware, and unit-testable. +- `scanner download` must verify digests/signatures, install containers locally (docker load), and log artefact metadata. +- `scan run` must execute the container against a directory, materialise artefacts in `ResultsDirectory`, and auto-upload them on success; `scan upload` is the manual retry path. +- Emit structured console logs (single line, UTC timestamps) and honour offline-first expectations—no hidden network calls. +- Mirror repository guidance: stay within `src/StellaOps.Cli` unless collaborating via documented handshakes. +- Update `TASKS.md` as states change (TODO → DOING → DONE/BLOCKED) and record added tests/fixtures alongside implementation notes. + +## Reference Materials +- `docs/ARCHITECTURE_FEEDSER.md` for database operations surface area. +- Backend OpenAPI/contract docs (once available) for job triggers and scanner endpoints. +- Existing module AGENTS/TASKS files for style and coordination cues. +- `docs/09_API_CLI_REFERENCE.md` (section 3) for the user-facing synopsis of the CLI verbs and flags. diff --git a/src/StellaOps.Cli/Commands/CommandFactory.cs b/src/StellaOps.Cli/Commands/CommandFactory.cs new file mode 100644 index 00000000..9f5b79f2 --- /dev/null +++ b/src/StellaOps.Cli/Commands/CommandFactory.cs @@ -0,0 +1,246 @@ +using System; +using System.CommandLine; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Cli.Configuration; + +namespace StellaOps.Cli.Commands; + +internal static class CommandFactory +{ + public static RootCommand Create(IServiceProvider services, StellaOpsCliOptions options, CancellationToken cancellationToken) + { + var verboseOption = new Option("--verbose", new[] { "-v" }) + { + Description = "Enable verbose logging output." + }; + + var root = new RootCommand("StellaOps command-line interface") + { + TreatUnmatchedTokensAsErrors = true + }; + root.Add(verboseOption); + + root.Add(BuildScannerCommand(services, verboseOption, cancellationToken)); + root.Add(BuildScanCommand(services, options, verboseOption, cancellationToken)); + root.Add(BuildDatabaseCommand(services, verboseOption, cancellationToken)); + root.Add(BuildConfigCommand(options)); + + return root; + } + + private static Command BuildScannerCommand(IServiceProvider services, Option verboseOption, CancellationToken cancellationToken) + { + var scanner = new Command("scanner", "Manage scanner artifacts and lifecycle."); + + var download = new Command("download", "Download the latest scanner bundle."); + var channelOption = new Option("--channel", new[] { "-c" }) + { + Description = "Scanner channel (stable, beta, nightly)." + }; + + var outputOption = new Option("--output") + { + Description = "Optional output path for the downloaded bundle." + }; + + var overwriteOption = new Option("--overwrite") + { + Description = "Overwrite existing bundle if present." + }; + + var noInstallOption = new Option("--no-install") + { + Description = "Skip installing the scanner container after download." + }; + + download.Add(channelOption); + download.Add(outputOption); + download.Add(overwriteOption); + download.Add(noInstallOption); + + download.SetAction((parseResult, _) => + { + var channel = parseResult.GetValue(channelOption) ?? "stable"; + var output = parseResult.GetValue(outputOption); + var overwrite = parseResult.GetValue(overwriteOption); + var install = !parseResult.GetValue(noInstallOption); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleScannerDownloadAsync(services, channel, output, overwrite, install, verbose, cancellationToken); + }); + + scanner.Add(download); + return scanner; + } + + private static Command BuildScanCommand(IServiceProvider services, StellaOpsCliOptions options, Option verboseOption, CancellationToken cancellationToken) + { + var scan = new Command("scan", "Execute scanners and manage scan outputs."); + + var run = new Command("run", "Execute a scanner bundle with the configured runner."); + var runnerOption = new Option("--runner") + { + Description = "Execution runtime (dotnet, self, docker)." + }; + var entryOption = new Option("--entry") + { + Description = "Path to the scanner entrypoint or Docker image.", + Required = true + }; + var targetOption = new Option("--target") + { + Description = "Directory to scan.", + Required = true + }; + + var argsArgument = new Argument("scanner-args") + { + Arity = ArgumentArity.ZeroOrMore + }; + + run.Add(runnerOption); + run.Add(entryOption); + run.Add(targetOption); + run.Add(argsArgument); + + run.SetAction((parseResult, _) => + { + var runner = parseResult.GetValue(runnerOption) ?? options.DefaultRunner; + var entry = parseResult.GetValue(entryOption) ?? string.Empty; + var target = parseResult.GetValue(targetOption) ?? string.Empty; + var forwardedArgs = parseResult.GetValue(argsArgument) ?? Array.Empty(); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleScannerRunAsync(services, runner, entry, target, forwardedArgs, verbose, cancellationToken); + }); + + var upload = new Command("upload", "Upload completed scan results to the backend."); + var fileOption = new Option("--file") + { + Description = "Path to the scan result artifact.", + Required = true + }; + upload.Add(fileOption); + upload.SetAction((parseResult, _) => + { + var file = parseResult.GetValue(fileOption) ?? string.Empty; + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleScanUploadAsync(services, file, verbose, cancellationToken); + }); + + scan.Add(run); + scan.Add(upload); + return scan; + } + + private static Command BuildDatabaseCommand(IServiceProvider services, Option verboseOption, CancellationToken cancellationToken) + { + var db = new Command("db", "Trigger Feedser database operations via backend jobs."); + + var fetch = new Command("fetch", "Trigger connector fetch/parse/map stages."); + var sourceOption = new Option("--source") + { + Description = "Connector source identifier (e.g. redhat, osv, vmware).", + Required = true + }; + var stageOption = new Option("--stage") + { + Description = "Stage to trigger: fetch, parse, or map." + }; + var modeOption = new Option("--mode") + { + Description = "Optional connector-specific mode (init, resume, cursor)." + }; + + fetch.Add(sourceOption); + fetch.Add(stageOption); + fetch.Add(modeOption); + fetch.SetAction((parseResult, _) => + { + var source = parseResult.GetValue(sourceOption) ?? string.Empty; + var stage = parseResult.GetValue(stageOption) ?? "fetch"; + var mode = parseResult.GetValue(modeOption); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleConnectorJobAsync(services, source, stage, mode, verbose, cancellationToken); + }); + + var merge = new Command("merge", "Run canonical merge reconciliation."); + merge.SetAction((parseResult, _) => + { + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleMergeJobAsync(services, verbose, cancellationToken); + }); + + var export = new Command("export", "Run Feedser export jobs."); + var formatOption = new Option("--format") + { + Description = "Export format: json or trivy-db." + }; + var deltaOption = new Option("--delta") + { + Description = "Request a delta export when supported." + }; + + export.Add(formatOption); + export.Add(deltaOption); + export.SetAction((parseResult, _) => + { + var format = parseResult.GetValue(formatOption) ?? "json"; + var delta = parseResult.GetValue(deltaOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExportJobAsync(services, format, delta, verbose, cancellationToken); + }); + + db.Add(fetch); + db.Add(merge); + db.Add(export); + return db; + } + + private static Command BuildConfigCommand(StellaOpsCliOptions options) + { + var config = new Command("config", "Inspect CLI configuration state."); + var show = new Command("show", "Display resolved configuration values."); + + show.SetAction((_, _) => + { + var lines = new[] + { + $"Backend URL: {MaskIfEmpty(options.BackendUrl)}", + $"API Key: {DescribeSecret(options.ApiKey)}", + $"Scanner Cache: {options.ScannerCacheDirectory}", + $"Results Directory: {options.ResultsDirectory}", + $"Default Runner: {options.DefaultRunner}" + }; + + foreach (var line in lines) + { + Console.WriteLine(line); + } + + return Task.CompletedTask; + }); + + config.Add(show); + return config; + } + + private static string MaskIfEmpty(string value) + => string.IsNullOrWhiteSpace(value) ? "" : value; + + private static string DescribeSecret(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ""; + } + + return value.Length switch + { + <= 4 => "****", + _ => $"{value[..2]}***{value[^2..]}" + }; + } +} diff --git a/src/StellaOps.Cli/Commands/CommandHandlers.cs b/src/StellaOps.Cli/Commands/CommandHandlers.cs new file mode 100644 index 00000000..5a963bab --- /dev/null +++ b/src/StellaOps.Cli/Commands/CommandHandlers.cs @@ -0,0 +1,323 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Telemetry; + +namespace StellaOps.Cli.Commands; + +internal static class CommandHandlers +{ + public static async Task HandleScannerDownloadAsync( + IServiceProvider services, + string channel, + string? output, + bool overwrite, + bool install, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("scanner-download"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.scanner.download", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "scanner download"); + activity?.SetTag("stellaops.cli.channel", channel); + using var duration = CliMetrics.MeasureCommandDuration("scanner download"); + + try + { + var result = await client.DownloadScannerAsync(channel, output ?? string.Empty, overwrite, verbose, cancellationToken).ConfigureAwait(false); + + if (result.FromCache) + { + logger.LogInformation("Using cached scanner at {Path}.", result.Path); + } + else + { + logger.LogInformation("Scanner downloaded to {Path} ({Size} bytes).", result.Path, result.SizeBytes); + } + + CliMetrics.RecordScannerDownload(channel, result.FromCache); + + if (install) + { + var installer = scope.ServiceProvider.GetRequiredService(); + await installer.InstallAsync(result.Path, verbose, cancellationToken).ConfigureAwait(false); + CliMetrics.RecordScannerInstall(channel); + } + + Environment.ExitCode = 0; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to download scanner bundle."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleScannerRunAsync( + IServiceProvider services, + string runner, + string entry, + string targetDirectory, + IReadOnlyList arguments, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var executor = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("scanner-run"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.scan.run", ActivityKind.Internal); + activity?.SetTag("stellaops.cli.command", "scan run"); + activity?.SetTag("stellaops.cli.runner", runner); + activity?.SetTag("stellaops.cli.entry", entry); + activity?.SetTag("stellaops.cli.target", targetDirectory); + using var duration = CliMetrics.MeasureCommandDuration("scan run"); + + try + { + var options = scope.ServiceProvider.GetRequiredService(); + var resultsDirectory = options.ResultsDirectory; + + var executionResult = await executor.RunAsync( + runner, + entry, + targetDirectory, + resultsDirectory, + arguments, + verbose, + cancellationToken).ConfigureAwait(false); + + Environment.ExitCode = executionResult.ExitCode; + CliMetrics.RecordScanRun(runner, executionResult.ExitCode); + + if (executionResult.ExitCode == 0) + { + var backend = scope.ServiceProvider.GetRequiredService(); + logger.LogInformation("Uploading scan artefact {Path}...", executionResult.ResultsPath); + await backend.UploadScanResultsAsync(executionResult.ResultsPath, cancellationToken).ConfigureAwait(false); + logger.LogInformation("Scan artefact uploaded."); + activity?.SetTag("stellaops.cli.results", executionResult.ResultsPath); + } + else + { + logger.LogWarning("Skipping automatic upload because scan exited with code {Code}.", executionResult.ExitCode); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Scanner execution failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleScanUploadAsync( + IServiceProvider services, + string file, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("scanner-upload"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.scan.upload", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "scan upload"); + activity?.SetTag("stellaops.cli.file", file); + using var duration = CliMetrics.MeasureCommandDuration("scan upload"); + + try + { + var path = Path.GetFullPath(file); + await client.UploadScanResultsAsync(path, cancellationToken).ConfigureAwait(false); + logger.LogInformation("Scan results uploaded successfully."); + Environment.ExitCode = 0; + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to upload scan results."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleConnectorJobAsync( + IServiceProvider services, + string source, + string stage, + string? mode, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("db-connector"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.db.fetch", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "db fetch"); + activity?.SetTag("stellaops.cli.source", source); + activity?.SetTag("stellaops.cli.stage", stage); + if (!string.IsNullOrWhiteSpace(mode)) + { + activity?.SetTag("stellaops.cli.mode", mode); + } + using var duration = CliMetrics.MeasureCommandDuration("db fetch"); + + try + { + var jobKind = $"source:{source}:{stage}"; + var parameters = new Dictionary(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(mode)) + { + parameters["mode"] = mode; + } + + await TriggerJobAsync(client, logger, jobKind, parameters, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError(ex, "Connector job failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleMergeJobAsync( + IServiceProvider services, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("db-merge"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.db.merge", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "db merge"); + using var duration = CliMetrics.MeasureCommandDuration("db merge"); + + try + { + await TriggerJobAsync(client, logger, "merge:reconcile", new Dictionary(StringComparer.Ordinal), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError(ex, "Merge job failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleExportJobAsync( + IServiceProvider services, + string format, + bool delta, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("db-export"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.db.export", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "db export"); + activity?.SetTag("stellaops.cli.format", format); + activity?.SetTag("stellaops.cli.delta", delta); + using var duration = CliMetrics.MeasureCommandDuration("db export"); + + try + { + var jobKind = format switch + { + "trivy-db" or "trivy" => "export:trivy-db", + _ => "export:json" + }; + + var parameters = new Dictionary(StringComparer.Ordinal) + { + ["delta"] = delta + }; + + await TriggerJobAsync(client, logger, jobKind, parameters, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError(ex, "Export job failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + private static async Task TriggerJobAsync( + IBackendOperationsClient client, + ILogger logger, + string jobKind, + IDictionary parameters, + CancellationToken cancellationToken) + { + JobTriggerResult result = await client.TriggerJobAsync(jobKind, parameters, cancellationToken).ConfigureAwait(false); + if (result.Success) + { + if (!string.IsNullOrWhiteSpace(result.Location)) + { + logger.LogInformation("Job accepted. Track status at {Location}.", result.Location); + } + else if (result.Run is not null) + { + logger.LogInformation("Job accepted. RunId: {RunId} Status: {Status}", result.Run.RunId, result.Run.Status); + } + else + { + logger.LogInformation("Job accepted."); + } + + Environment.ExitCode = 0; + } + else + { + logger.LogError("Job '{JobKind}' failed: {Message}", jobKind, result.Message); + Environment.ExitCode = 1; + } + } +} diff --git a/src/StellaOps.Cli/Configuration/CliBootstrapper.cs b/src/StellaOps.Cli/Configuration/CliBootstrapper.cs new file mode 100644 index 00000000..3c415cf7 --- /dev/null +++ b/src/StellaOps.Cli/Configuration/CliBootstrapper.cs @@ -0,0 +1,77 @@ +using System.Globalization; +using Microsoft.Extensions.Configuration; +using StellaOps.Configuration; + +namespace StellaOps.Cli.Configuration; + +public static class CliBootstrapper +{ + public static (StellaOpsCliOptions Options, IConfigurationRoot Configuration) Build(string[] args) + { + var bootstrap = StellaOpsConfigurationBootstrapper.Build(options => + { + options.BindingSection = "StellaOps"; + options.ConfigureBuilder = builder => + { + if (args.Length > 0) + { + builder.AddCommandLine(args); + } + }; + options.PostBind = (cliOptions, configuration) => + { + cliOptions.ApiKey = ResolveWithFallback(cliOptions.ApiKey, configuration, "API_KEY", "StellaOps:ApiKey", "ApiKey"); + cliOptions.BackendUrl = ResolveWithFallback(cliOptions.BackendUrl, configuration, "STELLAOPS_BACKEND_URL", "StellaOps:BackendUrl", "BackendUrl"); + cliOptions.ScannerSignaturePublicKeyPath = ResolveWithFallback(cliOptions.ScannerSignaturePublicKeyPath, configuration, "SCANNER_PUBLIC_KEY", "STELLAOPS_SCANNER_PUBLIC_KEY", "StellaOps:ScannerSignaturePublicKeyPath", "ScannerSignaturePublicKeyPath"); + + cliOptions.ApiKey = cliOptions.ApiKey?.Trim() ?? string.Empty; + cliOptions.BackendUrl = cliOptions.BackendUrl?.Trim() ?? string.Empty; + cliOptions.ScannerSignaturePublicKeyPath = cliOptions.ScannerSignaturePublicKeyPath?.Trim() ?? string.Empty; + + var attemptsRaw = ResolveWithFallback( + string.Empty, + configuration, + "SCANNER_DOWNLOAD_ATTEMPTS", + "STELLAOPS_SCANNER_DOWNLOAD_ATTEMPTS", + "StellaOps:ScannerDownloadAttempts", + "ScannerDownloadAttempts"); + + if (string.IsNullOrWhiteSpace(attemptsRaw)) + { + attemptsRaw = cliOptions.ScannerDownloadAttempts.ToString(CultureInfo.InvariantCulture); + } + + if (int.TryParse(attemptsRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedAttempts) && parsedAttempts > 0) + { + cliOptions.ScannerDownloadAttempts = parsedAttempts; + } + + if (cliOptions.ScannerDownloadAttempts <= 0) + { + cliOptions.ScannerDownloadAttempts = 3; + } + }; + }); + + return (bootstrap.Options, bootstrap.Configuration); + } + + private static string ResolveWithFallback(string currentValue, IConfiguration configuration, params string[] keys) + { + if (!string.IsNullOrWhiteSpace(currentValue)) + { + return currentValue; + } + + foreach (var key in keys) + { + var value = configuration[key]; + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + + return string.Empty; + } +} diff --git a/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs b/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs new file mode 100644 index 00000000..3a025190 --- /dev/null +++ b/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs @@ -0,0 +1,18 @@ +namespace StellaOps.Cli.Configuration; + +public sealed class StellaOpsCliOptions +{ + public string ApiKey { get; set; } = string.Empty; + + public string BackendUrl { get; set; } = string.Empty; + + public string ScannerCacheDirectory { get; set; } = "scanners"; + + public string ResultsDirectory { get; set; } = "results"; + + public string DefaultRunner { get; set; } = "docker"; + + public string ScannerSignaturePublicKeyPath { get; set; } = string.Empty; + + public int ScannerDownloadAttempts { get; set; } = 3; +} diff --git a/src/StellaOps.Cli/Program.cs b/src/StellaOps.Cli/Program.cs new file mode 100644 index 00000000..4ad61188 --- /dev/null +++ b/src/StellaOps.Cli/Program.cs @@ -0,0 +1,71 @@ +using System; +using System.CommandLine; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Cli.Commands; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services; +using StellaOps.Cli.Telemetry; + +namespace StellaOps.Cli; + +internal static class Program +{ + internal static async Task Main(string[] args) + { + var (options, configuration) = CliBootstrapper.Build(args); + + var services = new ServiceCollection(); + services.AddSingleton(configuration); + services.AddSingleton(options); + + var verbosityState = new VerbosityState(); + services.AddSingleton(verbosityState); + + services.AddLogging(builder => + { + builder.ClearProviders(); + builder.AddSimpleConsole(logOptions => + { + logOptions.TimestampFormat = "HH:mm:ss "; + logOptions.SingleLine = true; + }); + builder.AddFilter((category, level) => level >= verbosityState.MinimumLevel); + }); + + services.AddHttpClient(client => + { + client.Timeout = TimeSpan.FromMinutes(5); + if (!string.IsNullOrWhiteSpace(options.BackendUrl) && + Uri.TryCreate(options.BackendUrl, UriKind.Absolute, out var backendUri)) + { + client.BaseAddress = backendUri; + } + }); + + services.AddSingleton(); + services.AddSingleton(); + + await using var serviceProvider = services.BuildServiceProvider(); + using var cts = new CancellationTokenSource(); + Console.CancelKeyPress += (_, eventArgs) => + { + eventArgs.Cancel = true; + cts.Cancel(); + }; + + var rootCommand = CommandFactory.Create(serviceProvider, options, cts.Token); + var commandConfiguration = new CommandLineConfiguration(rootCommand); + var commandExit = await commandConfiguration.InvokeAsync(args, cts.Token).ConfigureAwait(false); + + var finalExit = Environment.ExitCode != 0 ? Environment.ExitCode : commandExit; + if (cts.IsCancellationRequested && finalExit == 0) + { + finalExit = 130; // Typical POSIX cancellation exit code + } + + return finalExit; + } +} diff --git a/src/StellaOps.Cli/Properties/AssemblyInfo.cs b/src/StellaOps.Cli/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..49f78a0c --- /dev/null +++ b/src/StellaOps.Cli/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Cli.Tests")] diff --git a/src/StellaOps.Cli/Services/BackendOperationsClient.cs b/src/StellaOps.Cli/Services/BackendOperationsClient.cs new file mode 100644 index 00000000..65d2c010 --- /dev/null +++ b/src/StellaOps.Cli/Services/BackendOperationsClient.cs @@ -0,0 +1,394 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Linq; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Services.Models.Transport; + +namespace StellaOps.Cli.Services; + +internal sealed class BackendOperationsClient : IBackendOperationsClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + private readonly HttpClient _httpClient; + private readonly StellaOpsCliOptions _options; + private readonly ILogger _logger; + + public BackendOperationsClient(HttpClient httpClient, StellaOpsCliOptions options, ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (!string.IsNullOrWhiteSpace(_options.BackendUrl) && httpClient.BaseAddress is null) + { + if (Uri.TryCreate(_options.BackendUrl, UriKind.Absolute, out var baseUri)) + { + httpClient.BaseAddress = baseUri; + } + } + } + + public async Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + channel = string.IsNullOrWhiteSpace(channel) ? "stable" : channel.Trim(); + outputPath = ResolveArtifactPath(outputPath, channel); + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); + + if (!overwrite && File.Exists(outputPath)) + { + var existing = new FileInfo(outputPath); + _logger.LogInformation("Scanner artifact already cached at {Path} ({Size} bytes).", outputPath, existing.Length); + return new ScannerArtifactResult(outputPath, existing.Length, true); + } + + var attempt = 0; + var maxAttempts = Math.Max(1, _options.ScannerDownloadAttempts); + + while (true) + { + attempt++; + try + { + using var request = CreateRequest(HttpMethod.Get, $"api/scanner/artifacts/{channel}"); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + return await ProcessScannerResponseAsync(response, outputPath, channel, verbose, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < maxAttempts) + { + var backoffSeconds = Math.Pow(2, attempt); + _logger.LogWarning(ex, "Scanner download attempt {Attempt}/{MaxAttempts} failed. Retrying in {Delay:F0}s...", attempt, maxAttempts, backoffSeconds); + await Task.Delay(TimeSpan.FromSeconds(backoffSeconds), cancellationToken).ConfigureAwait(false); + } + } + } + + private async Task ProcessScannerResponseAsync(HttpResponseMessage response, string outputPath, string channel, bool verbose, CancellationToken cancellationToken) + { + var tempFile = outputPath + ".tmp"; + await using (var payloadStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) + await using (var fileStream = File.Create(tempFile)) + { + await payloadStream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); + } + + var expectedDigest = ExtractHeaderValue(response.Headers, "X-StellaOps-Digest"); + var signatureHeader = ExtractHeaderValue(response.Headers, "X-StellaOps-Signature"); + + var digestHex = await ValidateDigestAsync(tempFile, expectedDigest, cancellationToken).ConfigureAwait(false); + await ValidateSignatureAsync(signatureHeader, digestHex, verbose, cancellationToken).ConfigureAwait(false); + + if (verbose) + { + var signatureNote = string.IsNullOrWhiteSpace(signatureHeader) ? "no signature" : "signature validated"; + _logger.LogDebug("Scanner digest sha256:{Digest} ({SignatureNote}).", digestHex, signatureNote); + } + + if (File.Exists(outputPath)) + { + File.Delete(outputPath); + } + + File.Move(tempFile, outputPath); + + PersistMetadata(outputPath, channel, digestHex, signatureHeader, response); + + var downloaded = new FileInfo(outputPath); + _logger.LogInformation("Scanner downloaded to {Path} ({Size} bytes).", outputPath, downloaded.Length); + + return new ScannerArtifactResult(outputPath, downloaded.Length, false); + } + + public async Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (!File.Exists(filePath)) + { + throw new FileNotFoundException("Scan result file not found.", filePath); + } + + using var content = new MultipartFormDataContent(); + await using var fileStream = File.OpenRead(filePath); + var streamContent = new StreamContent(fileStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + content.Add(streamContent, "file", Path.GetFileName(filePath)); + + var request = CreateRequest(HttpMethod.Post, "api/scanner/results"); + request.Content = content; + + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + _logger.LogInformation("Scan results uploaded from {Path}.", filePath); + } + + public async Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (string.IsNullOrWhiteSpace(jobKind)) + { + throw new ArgumentException("Job kind must be provided.", nameof(jobKind)); + } + + var requestBody = new JobTriggerRequest + { + Trigger = "cli", + Parameters = parameters is null ? new Dictionary(StringComparer.Ordinal) : new Dictionary(parameters, StringComparer.Ordinal) + }; + + var request = CreateRequest(HttpMethod.Post, $"jobs/{jobKind}"); + request.Content = JsonContent.Create(requestBody, options: SerializerOptions); + + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.Accepted) + { + JobRunResponse? run = null; + if (response.Content.Headers.ContentLength is > 0) + { + try + { + run = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to deserialize job run response for job kind {Kind}.", jobKind); + } + } + + var location = response.Headers.Location?.ToString(); + return new JobTriggerResult(true, "Accepted", location, run); + } + + var failureMessage = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + return new JobTriggerResult(false, failureMessage, null, null); + } + + private HttpRequestMessage CreateRequest(HttpMethod method, string relativeUri) + { + if (!Uri.TryCreate(relativeUri, UriKind.RelativeOrAbsolute, out var requestUri)) + { + throw new InvalidOperationException($"Invalid request URI '{relativeUri}'."); + } + + if (requestUri.IsAbsoluteUri) + { + // Nothing to normalize. + } + else + { + requestUri = new Uri(relativeUri.TrimStart('/'), UriKind.Relative); + } + + var request = new HttpRequestMessage(method, requestUri); + if (!string.IsNullOrWhiteSpace(_options.ApiKey)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ApiKey); + } + + return request; + } + + private void EnsureBackendConfigured() + { + if (_httpClient.BaseAddress is null) + { + throw new InvalidOperationException("Backend URL is not configured. Provide STELLAOPS_BACKEND_URL or configure appsettings."); + } + } + + private string ResolveArtifactPath(string outputPath, string channel) + { + if (!string.IsNullOrWhiteSpace(outputPath)) + { + return Path.GetFullPath(outputPath); + } + + var directory = string.IsNullOrWhiteSpace(_options.ScannerCacheDirectory) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(_options.ScannerCacheDirectory); + + Directory.CreateDirectory(directory); + var fileName = $"stellaops-scanner-{channel}.tar.gz"; + return Path.Combine(directory, fileName); + } + + private async Task CreateFailureMessageAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + var statusCode = (int)response.StatusCode; + var builder = new StringBuilder(); + builder.Append("Backend request failed with status "); + builder.Append(statusCode); + builder.Append(' '); + builder.Append(response.ReasonPhrase ?? "Unknown"); + + if (response.Content.Headers.ContentLength is > 0) + { + try + { + var problem = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false); + if (problem is not null) + { + if (!string.IsNullOrWhiteSpace(problem.Title)) + { + builder.AppendLine().Append(problem.Title); + } + + if (!string.IsNullOrWhiteSpace(problem.Detail)) + { + builder.AppendLine().Append(problem.Detail); + } + } + } + catch (JsonException) + { + var raw = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(raw)) + { + builder.AppendLine().Append(raw); + } + } + } + + return builder.ToString(); + } + + private static string? ExtractHeaderValue(HttpResponseHeaders headers, string name) + { + if (headers.TryGetValues(name, out var values)) + { + return values.FirstOrDefault(); + } + + return null; + } + + private async Task ValidateDigestAsync(string filePath, string? expectedDigest, CancellationToken cancellationToken) + { + string digestHex; + await using (var stream = File.OpenRead(filePath)) + { + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + digestHex = Convert.ToHexString(hash).ToLowerInvariant(); + } + + if (!string.IsNullOrWhiteSpace(expectedDigest)) + { + var normalized = NormalizeDigest(expectedDigest); + if (!normalized.Equals(digestHex, StringComparison.OrdinalIgnoreCase)) + { + File.Delete(filePath); + throw new InvalidOperationException($"Scanner digest mismatch. Expected sha256:{normalized}, calculated sha256:{digestHex}."); + } + } + else + { + _logger.LogWarning("Scanner download missing X-StellaOps-Digest header; relying on computed digest only."); + } + + return digestHex; + } + + private static string NormalizeDigest(string digest) + { + if (digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + return digest[7..]; + } + + return digest; + } + + private async Task ValidateSignatureAsync(string? signatureHeader, string digestHex, bool verbose, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(_options.ScannerSignaturePublicKeyPath)) + { + if (!string.IsNullOrWhiteSpace(signatureHeader)) + { + _logger.LogDebug("Signature header present but no public key configured; skipping validation."); + } + return; + } + + if (string.IsNullOrWhiteSpace(signatureHeader)) + { + throw new InvalidOperationException("Scanner signature missing while a public key is configured."); + } + + var publicKeyPath = Path.GetFullPath(_options.ScannerSignaturePublicKeyPath); + if (!File.Exists(publicKeyPath)) + { + throw new FileNotFoundException("Scanner signature public key not found.", publicKeyPath); + } + + var signatureBytes = Convert.FromBase64String(signatureHeader); + var digestBytes = Convert.FromHexString(digestHex); + + var pem = await File.ReadAllTextAsync(publicKeyPath, cancellationToken).ConfigureAwait(false); + using var rsa = RSA.Create(); + rsa.ImportFromPem(pem); + + var valid = rsa.VerifyHash(digestBytes, signatureBytes, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + if (!valid) + { + throw new InvalidOperationException("Scanner signature validation failed."); + } + + if (verbose) + { + _logger.LogDebug("Scanner signature validated using key {KeyPath}.", publicKeyPath); + } + } + + private void PersistMetadata(string outputPath, string channel, string digestHex, string? signatureHeader, HttpResponseMessage response) + { + var metadata = new + { + channel, + digest = $"sha256:{digestHex}", + signature = signatureHeader, + downloadedAt = DateTimeOffset.UtcNow, + source = response.RequestMessage?.RequestUri?.ToString(), + sizeBytes = new FileInfo(outputPath).Length, + headers = new + { + etag = response.Headers.ETag?.Tag, + lastModified = response.Content.Headers.LastModified, + contentType = response.Content.Headers.ContentType?.ToString() + } + }; + + var metadataPath = outputPath + ".metadata.json"; + var json = JsonSerializer.Serialize(metadata, new JsonSerializerOptions + { + WriteIndented = true + }); + + File.WriteAllText(metadataPath, json); + } +} diff --git a/src/StellaOps.Cli/Services/IBackendOperationsClient.cs b/src/StellaOps.Cli/Services/IBackendOperationsClient.cs new file mode 100644 index 00000000..3a285643 --- /dev/null +++ b/src/StellaOps.Cli/Services/IBackendOperationsClient.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services.Models; + +namespace StellaOps.Cli.Services; + +internal interface IBackendOperationsClient +{ + Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken); + + Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken); + + Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Cli/Services/IScannerExecutor.cs b/src/StellaOps.Cli/Services/IScannerExecutor.cs new file mode 100644 index 00000000..cfb9377c --- /dev/null +++ b/src/StellaOps.Cli/Services/IScannerExecutor.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Cli.Services; + +internal interface IScannerExecutor +{ + Task RunAsync( + string runner, + string entry, + string targetDirectory, + string resultsDirectory, + IReadOnlyList arguments, + bool verbose, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Cli/Services/IScannerInstaller.cs b/src/StellaOps.Cli/Services/IScannerInstaller.cs new file mode 100644 index 00000000..d6a3ec9d --- /dev/null +++ b/src/StellaOps.Cli/Services/IScannerInstaller.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Cli.Services; + +internal interface IScannerInstaller +{ + Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Cli/Services/Models/JobTriggerResult.cs b/src/StellaOps.Cli/Services/Models/JobTriggerResult.cs new file mode 100644 index 00000000..1ff4dded --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/JobTriggerResult.cs @@ -0,0 +1,9 @@ +using StellaOps.Cli.Services.Models.Transport; + +namespace StellaOps.Cli.Services.Models; + +internal sealed record JobTriggerResult( + bool Success, + string Message, + string? Location, + JobRunResponse? Run); diff --git a/src/StellaOps.Cli/Services/Models/ScannerArtifactResult.cs b/src/StellaOps.Cli/Services/Models/ScannerArtifactResult.cs new file mode 100644 index 00000000..8ef30e1b --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/ScannerArtifactResult.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Cli.Services.Models; + +internal sealed record ScannerArtifactResult(string Path, long SizeBytes, bool FromCache); diff --git a/src/StellaOps.Cli/Services/Models/Transport/JobRunResponse.cs b/src/StellaOps.Cli/Services/Models/Transport/JobRunResponse.cs new file mode 100644 index 00000000..00725878 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/Transport/JobRunResponse.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models.Transport; + +internal sealed class JobRunResponse +{ + public Guid RunId { get; set; } + + public string Kind { get; set; } = string.Empty; + + public string Status { get; set; } = string.Empty; + + public string Trigger { get; set; } = string.Empty; + + public DateTimeOffset CreatedAt { get; set; } + + public DateTimeOffset? StartedAt { get; set; } + + public DateTimeOffset? CompletedAt { get; set; } + + public string? Error { get; set; } + + public TimeSpan? Duration { get; set; } + + public IReadOnlyDictionary Parameters { get; set; } = new Dictionary(StringComparer.Ordinal); +} diff --git a/src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs b/src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs new file mode 100644 index 00000000..e071112c --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models.Transport; + +internal sealed class JobTriggerRequest +{ + public string Trigger { get; set; } = "cli"; + + public Dictionary Parameters { get; set; } = new(StringComparer.Ordinal); +} diff --git a/src/StellaOps.Cli/Services/Models/Transport/ProblemDocument.cs b/src/StellaOps.Cli/Services/Models/Transport/ProblemDocument.cs new file mode 100644 index 00000000..e6f4e152 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/Transport/ProblemDocument.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models.Transport; + +internal sealed class ProblemDocument +{ + public string? Type { get; set; } + + public string? Title { get; set; } + + public string? Detail { get; set; } + + public int? Status { get; set; } + + public string? Instance { get; set; } + + public Dictionary? Extensions { get; set; } +} diff --git a/src/StellaOps.Cli/Services/ScannerExecutionResult.cs b/src/StellaOps.Cli/Services/ScannerExecutionResult.cs new file mode 100644 index 00000000..aedc76fa --- /dev/null +++ b/src/StellaOps.Cli/Services/ScannerExecutionResult.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Cli.Services; + +internal sealed record ScannerExecutionResult(int ExitCode, string ResultsPath); diff --git a/src/StellaOps.Cli/Services/ScannerExecutor.cs b/src/StellaOps.Cli/Services/ScannerExecutor.cs new file mode 100644 index 00000000..fc6233fd --- /dev/null +++ b/src/StellaOps.Cli/Services/ScannerExecutor.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Cli.Services; + +internal sealed class ScannerExecutor : IScannerExecutor +{ + private readonly ILogger _logger; + + public ScannerExecutor(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task RunAsync( + string runner, + string entry, + string targetDirectory, + string resultsDirectory, + IReadOnlyList arguments, + bool verbose, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(targetDirectory)) + { + throw new ArgumentException("Target directory must be provided.", nameof(targetDirectory)); + } + + runner = string.IsNullOrWhiteSpace(runner) ? "docker" : runner.Trim().ToLowerInvariant(); + entry = entry?.Trim() ?? string.Empty; + + var normalizedTarget = Path.GetFullPath(targetDirectory); + if (!Directory.Exists(normalizedTarget)) + { + throw new DirectoryNotFoundException($"Scan target directory '{normalizedTarget}' does not exist."); + } + + resultsDirectory = string.IsNullOrWhiteSpace(resultsDirectory) + ? Path.Combine(Directory.GetCurrentDirectory(), "scan-results") + : Path.GetFullPath(resultsDirectory); + + Directory.CreateDirectory(resultsDirectory); + var executionTimestamp = DateTimeOffset.UtcNow; + var baselineFiles = Directory.GetFiles(resultsDirectory, "*", SearchOption.AllDirectories); + var baseline = new HashSet(baselineFiles, StringComparer.OrdinalIgnoreCase); + + var startInfo = BuildProcessStartInfo(runner, entry, normalizedTarget, resultsDirectory, arguments); + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + var stdout = new List(); + var stderr = new List(); + + process.OutputDataReceived += (_, args) => + { + if (args.Data is null) + { + return; + } + + stdout.Add(args.Data); + if (verbose) + { + _logger.LogInformation("[scan] {Line}", args.Data); + } + }; + + process.ErrorDataReceived += (_, args) => + { + if (args.Data is null) + { + return; + } + + stderr.Add(args.Data); + _logger.LogError("[scan] {Line}", args.Data); + }; + + _logger.LogInformation("Launching scanner via {Runner} (entry: {Entry})...", runner, entry); + if (!process.Start()) + { + throw new InvalidOperationException("Failed to start scanner process."); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + + if (process.ExitCode == 0) + { + _logger.LogInformation("Scanner completed successfully."); + } + else + { + _logger.LogWarning("Scanner exited with code {Code}.", process.ExitCode); + } + + var resultsPath = ResolveResultsPath(resultsDirectory, executionTimestamp, baseline); + if (string.IsNullOrWhiteSpace(resultsPath)) + { + resultsPath = CreatePlaceholderResult(resultsDirectory); + } + + return new ScannerExecutionResult(process.ExitCode, resultsPath); + } + + private ProcessStartInfo BuildProcessStartInfo( + string runner, + string entry, + string targetDirectory, + string resultsDirectory, + IReadOnlyList args) + { + return runner switch + { + "self" or "native" => BuildNativeStartInfo(entry, args), + "dotnet" => BuildDotNetStartInfo(entry, args), + "docker" => BuildDockerStartInfo(entry, targetDirectory, resultsDirectory, args), + _ => BuildCustomRunnerStartInfo(runner, entry, args) + }; + } + + private static ProcessStartInfo BuildNativeStartInfo(string binaryPath, IReadOnlyList args) + { + if (string.IsNullOrWhiteSpace(binaryPath) || !File.Exists(binaryPath)) + { + throw new FileNotFoundException("Scanner entrypoint not found.", binaryPath); + } + + var startInfo = new ProcessStartInfo + { + FileName = binaryPath, + WorkingDirectory = Directory.GetCurrentDirectory() + }; + + foreach (var argument in args) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + startInfo.UseShellExecute = false; + + return startInfo; + } + + private static ProcessStartInfo BuildDotNetStartInfo(string binaryPath, IReadOnlyList args) + { + var startInfo = new ProcessStartInfo + { + FileName = "dotnet", + WorkingDirectory = Directory.GetCurrentDirectory() + }; + + startInfo.ArgumentList.Add(binaryPath); + foreach (var argument in args) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + startInfo.UseShellExecute = false; + + return startInfo; + } + + private static ProcessStartInfo BuildDockerStartInfo(string image, string targetDirectory, string resultsDirectory, IReadOnlyList args) + { + if (string.IsNullOrWhiteSpace(image)) + { + throw new ArgumentException("Docker image must be provided when runner is 'docker'.", nameof(image)); + } + + var cwd = Directory.GetCurrentDirectory(); + + var startInfo = new ProcessStartInfo + { + FileName = "docker", + WorkingDirectory = cwd + }; + + startInfo.ArgumentList.Add("run"); + startInfo.ArgumentList.Add("--rm"); + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add($"{cwd}:{cwd}"); + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add($"{targetDirectory}:/scan-target:ro"); + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add($"{resultsDirectory}:/scan-results"); + startInfo.ArgumentList.Add("-w"); + startInfo.ArgumentList.Add(cwd); + startInfo.ArgumentList.Add(image); + startInfo.ArgumentList.Add("--target"); + startInfo.ArgumentList.Add("/scan-target"); + startInfo.ArgumentList.Add("--output"); + startInfo.ArgumentList.Add("/scan-results/scan.json"); + + foreach (var argument in args) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + startInfo.UseShellExecute = false; + + return startInfo; + } + + private static ProcessStartInfo BuildCustomRunnerStartInfo(string runner, string entry, IReadOnlyList args) + { + var startInfo = new ProcessStartInfo + { + FileName = runner, + WorkingDirectory = Directory.GetCurrentDirectory() + }; + + if (!string.IsNullOrWhiteSpace(entry)) + { + startInfo.ArgumentList.Add(entry); + } + + foreach (var argument in args) + { + startInfo.ArgumentList.Add(argument); + } + + startInfo.RedirectStandardError = true; + startInfo.RedirectStandardOutput = true; + startInfo.UseShellExecute = false; + + return startInfo; + } + + private static string ResolveResultsPath(string resultsDirectory, DateTimeOffset startTimestamp, HashSet baseline) + { + var candidates = Directory.GetFiles(resultsDirectory, "*", SearchOption.AllDirectories); + string? newest = null; + DateTimeOffset newestTimestamp = startTimestamp; + + foreach (var candidate in candidates) + { + if (baseline.Contains(candidate)) + { + continue; + } + + var info = new FileInfo(candidate); + if (info.LastWriteTimeUtc >= newestTimestamp) + { + newestTimestamp = info.LastWriteTimeUtc; + newest = candidate; + } + } + + return newest ?? string.Empty; + } + + private static string CreatePlaceholderResult(string resultsDirectory) + { + var fileName = $"scan-{DateTimeOffset.UtcNow:yyyyMMddHHmmss}.json"; + var path = Path.Combine(resultsDirectory, fileName); + File.WriteAllText(path, "{\"status\":\"placeholder\"}"); + return path; + } +} diff --git a/src/StellaOps.Cli/Services/ScannerInstaller.cs b/src/StellaOps.Cli/Services/ScannerInstaller.cs new file mode 100644 index 00000000..97b3a7c4 --- /dev/null +++ b/src/StellaOps.Cli/Services/ScannerInstaller.cs @@ -0,0 +1,79 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Cli.Services; + +internal sealed class ScannerInstaller : IScannerInstaller +{ + private readonly ILogger _logger; + + public ScannerInstaller(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(artifactPath) || !File.Exists(artifactPath)) + { + throw new FileNotFoundException("Scanner artifact not found for installation.", artifactPath); + } + + // Current implementation assumes docker-based scanner bundle. + var processInfo = new ProcessStartInfo + { + FileName = "docker", + ArgumentList = { "load", "-i", artifactPath }, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + using var process = new Process { StartInfo = processInfo, EnableRaisingEvents = true }; + + process.OutputDataReceived += (_, args) => + { + if (args.Data is null) + { + return; + } + + if (verbose) + { + _logger.LogInformation("[install] {Line}", args.Data); + } + }; + + process.ErrorDataReceived += (_, args) => + { + if (args.Data is null) + { + return; + } + + _logger.LogError("[install] {Line}", args.Data); + }; + + _logger.LogInformation("Installing scanner container from {Path}...", artifactPath); + if (!process.Start()) + { + throw new InvalidOperationException("Failed to start container installation process."); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"Container installation failed with exit code {process.ExitCode}."); + } + + _logger.LogInformation("Scanner container installed successfully."); + } +} diff --git a/src/StellaOps.Cli/StellaOps.Cli.csproj b/src/StellaOps.Cli/StellaOps.Cli.csproj new file mode 100644 index 00000000..8336508f --- /dev/null +++ b/src/StellaOps.Cli/StellaOps.Cli.csproj @@ -0,0 +1,41 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + diff --git a/src/StellaOps.Cli/TASKS.md b/src/StellaOps.Cli/TASKS.md new file mode 100644 index 00000000..2bca2a50 --- /dev/null +++ b/src/StellaOps.Cli/TASKS.md @@ -0,0 +1,9 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Bootstrap configuration fallback (env → appsettings{{.json/.yaml}})|DevEx/CLI|Core|**DONE** – CLI loads `API_KEY`/`STELLAOPS_BACKEND_URL` from environment or local settings, defaulting to empty strings when unset.| +|Introduce command host & routing skeleton|DevEx/CLI|Configuration|**DONE** – System.CommandLine (v2.0.0-beta5) router stitched with `scanner`, `scan`, `db`, and `config` verbs.| +|Scanner artifact download/install commands|Ops Integrator|Backend contracts|**DONE** – `scanner download` caches bundles, validates SHA-256 (plus optional RSA signature), installs via `docker load`, persists metadata, and retries with exponential backoff.| +|Scan execution & result upload workflow|Ops Integrator, QA|Scanner cmd|**DONE** – `scan run` drives container scans against directories, emits artefacts in `ResultsDirectory`, auto-uploads on success, and `scan upload` covers manual retries.| +|Feedser DB operations passthrough|DevEx/CLI|Backend, Feedser APIs|**DONE** – `db fetch|merge|export` trigger `/jobs/*` endpoints with parameter binding and consistent exit codes.| +|CLI observability & tests|QA|Command host|**DONE** – Added console logging defaults & configuration bootstrap tests; future metrics hooks tracked separately.| diff --git a/src/StellaOps.Cli/Telemetry/CliActivitySource.cs b/src/StellaOps.Cli/Telemetry/CliActivitySource.cs new file mode 100644 index 00000000..08386eec --- /dev/null +++ b/src/StellaOps.Cli/Telemetry/CliActivitySource.cs @@ -0,0 +1,8 @@ +using System.Diagnostics; + +namespace StellaOps.Cli.Telemetry; + +internal static class CliActivitySource +{ + public static readonly ActivitySource Instance = new("StellaOps.Cli"); +} diff --git a/src/StellaOps.Cli/Telemetry/CliMetrics.cs b/src/StellaOps.Cli/Telemetry/CliMetrics.cs new file mode 100644 index 00000000..00f3ce49 --- /dev/null +++ b/src/StellaOps.Cli/Telemetry/CliMetrics.cs @@ -0,0 +1,62 @@ +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Cli.Telemetry; + +internal static class CliMetrics +{ + private static readonly Meter Meter = new("StellaOps.Cli", "1.0.0"); + + private static readonly Counter ScannerDownloadCounter = Meter.CreateCounter("stellaops.cli.scanner.download.count"); + private static readonly Counter ScannerInstallCounter = Meter.CreateCounter("stellaops.cli.scanner.install.count"); + private static readonly Counter ScanRunCounter = Meter.CreateCounter("stellaops.cli.scan.run.count"); + private static readonly Histogram CommandDurationHistogram = Meter.CreateHistogram("stellaops.cli.command.duration.ms"); + + public static void RecordScannerDownload(string channel, bool fromCache) + => ScannerDownloadCounter.Add(1, new KeyValuePair[] + { + new("channel", channel), + new("cache", fromCache ? "hit" : "miss") + }); + + public static void RecordScannerInstall(string channel) + => ScannerInstallCounter.Add(1, new KeyValuePair[] { new("channel", channel) }); + + public static void RecordScanRun(string runner, int exitCode) + => ScanRunCounter.Add(1, new KeyValuePair[] + { + new("runner", runner), + new("exit_code", exitCode) + }); + + public static IDisposable MeasureCommandDuration(string command) + { + var start = DateTime.UtcNow; + return new DurationScope(command, start); + } + + private sealed class DurationScope : IDisposable + { + private readonly string _command; + private readonly DateTime _start; + private bool _disposed; + + public DurationScope(string command, DateTime start) + { + _command = command; + _start = start; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + var elapsed = (DateTime.UtcNow - _start).TotalMilliseconds; + CommandDurationHistogram.Record(elapsed, new KeyValuePair[] { new("command", _command) }); + } + } +} diff --git a/src/StellaOps.Cli/Telemetry/VerbosityState.cs b/src/StellaOps.Cli/Telemetry/VerbosityState.cs new file mode 100644 index 00000000..90781765 --- /dev/null +++ b/src/StellaOps.Cli/Telemetry/VerbosityState.cs @@ -0,0 +1,8 @@ +using Microsoft.Extensions.Logging; + +namespace StellaOps.Cli.Telemetry; + +internal sealed class VerbosityState +{ + public LogLevel MinimumLevel { get; set; } = LogLevel.Information; +} diff --git a/src/StellaOps.Cli/appsettings.json b/src/StellaOps.Cli/appsettings.json new file mode 100644 index 00000000..a0d35327 --- /dev/null +++ b/src/StellaOps.Cli/appsettings.json @@ -0,0 +1,11 @@ +{ + "StellaOps": { + "ApiKey": "", + "BackendUrl": "", + "ScannerCacheDirectory": "scanners", + "ResultsDirectory": "results", + "DefaultRunner": "dotnet", + "ScannerSignaturePublicKeyPath": "", + "ScannerDownloadAttempts": 3 + } +} diff --git a/src/StellaOps.Configuration/StellaOps.Configuration.csproj b/src/StellaOps.Configuration/StellaOps.Configuration.csproj new file mode 100644 index 00000000..f30b1924 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOps.Configuration.csproj @@ -0,0 +1,18 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs b/src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs new file mode 100644 index 00000000..dc36d024 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsBootstrapOptions.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Configuration; + +public sealed class StellaOpsBootstrapOptions + where TOptions : class, new() +{ + public StellaOpsBootstrapOptions() + { + ConfigurationOptions = new StellaOpsConfigurationOptions(); + } + + internal StellaOpsConfigurationOptions ConfigurationOptions { get; } + + public string? BasePath + { + get => ConfigurationOptions.BasePath; + set => ConfigurationOptions.BasePath = value; + } + + public bool IncludeJsonFiles + { + get => ConfigurationOptions.IncludeJsonFiles; + set => ConfigurationOptions.IncludeJsonFiles = value; + } + + public bool IncludeYamlFiles + { + get => ConfigurationOptions.IncludeYamlFiles; + set => ConfigurationOptions.IncludeYamlFiles = value; + } + + public bool IncludeEnvironmentVariables + { + get => ConfigurationOptions.IncludeEnvironmentVariables; + set => ConfigurationOptions.IncludeEnvironmentVariables = value; + } + + public string? EnvironmentPrefix + { + get => ConfigurationOptions.EnvironmentPrefix; + set => ConfigurationOptions.EnvironmentPrefix = value; + } + + public IList JsonFiles => ConfigurationOptions.JsonFiles; + + public IList YamlFiles => ConfigurationOptions.YamlFiles; + + public string? BindingSection + { + get => ConfigurationOptions.BindingSection; + set => ConfigurationOptions.BindingSection = value; + } + + public Action? ConfigureBuilder + { + get => ConfigurationOptions.ConfigureBuilder; + set => ConfigurationOptions.ConfigureBuilder = value; + } + + public Action? PostBind { get; set; } +} diff --git a/src/StellaOps.Configuration/StellaOpsConfigurationBootstrapper.cs b/src/StellaOps.Configuration/StellaOpsConfigurationBootstrapper.cs new file mode 100644 index 00000000..a0caf0a9 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsConfigurationBootstrapper.cs @@ -0,0 +1,106 @@ +using System; +using Microsoft.Extensions.Configuration; +using NetEscapades.Configuration.Yaml; + +namespace StellaOps.Configuration; + +public static class StellaOpsConfigurationBootstrapper +{ + public static StellaOpsConfigurationContext Build( + Action>? configure = null) + where TOptions : class, new() + { + var bootstrapOptions = new StellaOpsBootstrapOptions(); + configure?.Invoke(bootstrapOptions); + + var configurationOptions = bootstrapOptions.ConfigurationOptions; + var builder = new ConfigurationBuilder(); + + if (!string.IsNullOrWhiteSpace(configurationOptions.BasePath)) + { + builder.SetBasePath(configurationOptions.BasePath!); + } + + if (configurationOptions.IncludeJsonFiles) + { + foreach (var file in configurationOptions.JsonFiles) + { + builder.AddJsonFile(file.Path, optional: file.Optional, reloadOnChange: file.ReloadOnChange); + } + } + + if (configurationOptions.IncludeYamlFiles) + { + foreach (var file in configurationOptions.YamlFiles) + { + builder.AddYamlFile(file.Path, optional: file.Optional); + } + } + + configurationOptions.ConfigureBuilder?.Invoke(builder); + + if (configurationOptions.IncludeEnvironmentVariables) + { + builder.AddEnvironmentVariables(configurationOptions.EnvironmentPrefix); + } + + var configuration = builder.Build(); + + IConfiguration bindingSource; + if (string.IsNullOrWhiteSpace(configurationOptions.BindingSection)) + { + bindingSource = configuration; + } + else + { + bindingSource = configuration.GetSection(configurationOptions.BindingSection!); + } + + var options = new TOptions(); + bindingSource.Bind(options); + + bootstrapOptions.PostBind?.Invoke(options, configuration); + + return new StellaOpsConfigurationContext(configuration, options); + } + + public static IConfigurationBuilder AddStellaOpsDefaults( + this IConfigurationBuilder builder, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + var options = new StellaOpsConfigurationOptions(); + configure?.Invoke(options); + + if (!string.IsNullOrWhiteSpace(options.BasePath)) + { + builder.SetBasePath(options.BasePath!); + } + + if (options.IncludeJsonFiles) + { + foreach (var file in options.JsonFiles) + { + builder.AddJsonFile(file.Path, optional: file.Optional, reloadOnChange: file.ReloadOnChange); + } + } + + if (options.IncludeYamlFiles) + { + foreach (var file in options.YamlFiles) + { + builder.AddYamlFile(file.Path, optional: file.Optional); + } + } + + options.ConfigureBuilder?.Invoke(builder); + + if (options.IncludeEnvironmentVariables) + { + builder.AddEnvironmentVariables(options.EnvironmentPrefix); + } + + return builder; + } +} diff --git a/src/StellaOps.Configuration/StellaOpsConfigurationContext.cs b/src/StellaOps.Configuration/StellaOpsConfigurationContext.cs new file mode 100644 index 00000000..fb7a05cf --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsConfigurationContext.cs @@ -0,0 +1,18 @@ +using System; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Configuration; + +public sealed class StellaOpsConfigurationContext + where TOptions : class, new() +{ + public StellaOpsConfigurationContext(IConfigurationRoot configuration, TOptions options) + { + Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + Options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public IConfigurationRoot Configuration { get; } + + public TOptions Options { get; } +} diff --git a/src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs b/src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs new file mode 100644 index 00000000..dee819d0 --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsConfigurationOptions.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Configuration; + +/// +/// Defines how default StellaOps configuration sources are composed. +/// +public sealed class StellaOpsConfigurationOptions +{ + public string? BasePath { get; set; } = Directory.GetCurrentDirectory(); + + public bool IncludeJsonFiles { get; set; } = true; + + public bool IncludeYamlFiles { get; set; } = true; + + public bool IncludeEnvironmentVariables { get; set; } = true; + + public string? EnvironmentPrefix { get; set; } + + public IList JsonFiles { get; } = new List + { + new("appsettings.json", true, false), + new("appsettings.local.json", true, false) + }; + + public IList YamlFiles { get; } = new List + { + new("appsettings.yaml", true), + new("appsettings.local.yaml", true) + }; + + /// + /// Optional hook to register additional configuration sources (e.g. module-specific YAML files). + /// + public Action? ConfigureBuilder { get; set; } + + /// + /// Optional configuration section name used when binding strongly typed options. + /// Null or empty indicates the root. + /// + public string? BindingSection { get; set; } +} + +public sealed record JsonConfigurationFile(string Path, bool Optional = true, bool ReloadOnChange = false); + +public sealed record YamlConfigurationFile(string Path, bool Optional = true); diff --git a/src/StellaOps.Configuration/StellaOpsOptionsBinder.cs b/src/StellaOps.Configuration/StellaOpsOptionsBinder.cs new file mode 100644 index 00000000..c34faaab --- /dev/null +++ b/src/StellaOps.Configuration/StellaOpsOptionsBinder.cs @@ -0,0 +1,26 @@ +using System; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Configuration; + +public static class StellaOpsOptionsBinder +{ + public static TOptions BindOptions( + this IConfiguration configuration, + string? section = null, + Action? postConfigure = null) + where TOptions : class, new() + { + ArgumentNullException.ThrowIfNull(configuration); + + var options = new TOptions(); + var bindingSource = string.IsNullOrWhiteSpace(section) + ? configuration + : configuration.GetSection(section); + + bindingSource.Bind(options); + postConfigure?.Invoke(options, configuration); + + return options; + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs b/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs index c636457c..7b75e8a4 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs +++ b/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs @@ -1,6 +1,7 @@ using System; using System.Globalization; using System.IO; +using System.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Feedser.Storage.Mongo.Advisories; @@ -64,7 +65,13 @@ public sealed class JsonFeedExporter : IFeedExporter result.AdvisoryCount, digest); - if (existingState is not null && string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal)) + var manifest = result.Files + .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest)) + .ToArray(); + + if (existingState is not null + && existingState.Files.Count > 0 + && string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal)) { _logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId); TryDeleteDirectory(result.ExportDirectory); @@ -90,6 +97,7 @@ public sealed class JsonFeedExporter : IFeedExporter targetRepository: _options.TargetRepository, exporterVersion: _exporterVersion, resetBaseline: resetBaseline, + manifest: manifest, cancellationToken: cancellationToken).ConfigureAwait(false); await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs index 59de7ddb..0ef7486a 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs @@ -10,19 +10,22 @@ public sealed class TrivyDbExportPlannerTests public void CreatePlan_ReturnsFullWhenStateMissing() { var planner = new TrivyDbExportPlanner(); - var plan = planner.CreatePlan(existingState: null, treeDigest: "sha256:abcd"); + var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; + var plan = planner.CreatePlan(existingState: null, treeDigest: "sha256:abcd", manifest); Assert.Equal(TrivyDbExportMode.Full, plan.Mode); Assert.Equal("sha256:abcd", plan.TreeDigest); Assert.Null(plan.BaseExportId); Assert.Null(plan.BaseManifestDigest); Assert.True(plan.ResetBaseline); + Assert.Equal(manifest, plan.Manifest); } [Fact] public void CreatePlan_ReturnsSkipWhenCursorMatches() { var planner = new TrivyDbExportPlanner(); + var existingManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; var state = new ExportStateRecord( Id: TrivyDbFeedExporter.ExporterId, BaseExportId: "20240810T000000Z", @@ -32,21 +35,25 @@ public sealed class TrivyDbExportPlannerTests ExportCursor: "sha256:unchanged", TargetRepository: "feedser/trivy", ExporterVersion: "1.0", - UpdatedAt: DateTimeOffset.UtcNow); + UpdatedAt: DateTimeOffset.UtcNow, + Files: existingManifest); - var plan = planner.CreatePlan(state, "sha256:unchanged"); + var plan = planner.CreatePlan(state, "sha256:unchanged", existingManifest); Assert.Equal(TrivyDbExportMode.Skip, plan.Mode); Assert.Equal("sha256:unchanged", plan.TreeDigest); Assert.Equal("20240810T000000Z", plan.BaseExportId); Assert.Equal("sha256:base", plan.BaseManifestDigest); Assert.False(plan.ResetBaseline); + Assert.Empty(plan.ChangedFiles); + Assert.Empty(plan.RemovedPaths); } [Fact] public void CreatePlan_ReturnsFullWhenCursorDiffers() { var planner = new TrivyDbExportPlanner(); + var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; var state = new ExportStateRecord( Id: TrivyDbFeedExporter.ExporterId, BaseExportId: "20240810T000000Z", @@ -56,20 +63,23 @@ public sealed class TrivyDbExportPlannerTests ExportCursor: "sha256:old", TargetRepository: "feedser/trivy", ExporterVersion: "1.0", - UpdatedAt: DateTimeOffset.UtcNow); + UpdatedAt: DateTimeOffset.UtcNow, + Files: manifest); - var plan = planner.CreatePlan(state, "sha256:new"); + var newManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:b") }; + var plan = planner.CreatePlan(state, "sha256:new", newManifest); - Assert.Equal(TrivyDbExportMode.Full, plan.Mode); + Assert.Equal(TrivyDbExportMode.Delta, plan.Mode); Assert.Equal("sha256:new", plan.TreeDigest); Assert.Equal("20240810T000000Z", plan.BaseExportId); Assert.Equal("sha256:base", plan.BaseManifestDigest); Assert.False(plan.ResetBaseline); + Assert.Single(plan.ChangedFiles); var deltaState = state with { LastDeltaDigest = "sha256:delta" }; - var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer"); + var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer", newManifest); - Assert.Equal(TrivyDbExportMode.Full, deltaPlan.Mode); + Assert.Equal(TrivyDbExportMode.Delta, deltaPlan.Mode); Assert.True(deltaPlan.ResetBaseline); } } diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs index befce26b..924cad10 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -315,7 +316,8 @@ public sealed class TrivyDbFeedExporterTests : IDisposable ExportCursor: "sha256:old", TargetRepository: "registry.example/trivy", ExporterVersion: "0.9.0", - UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30)); + UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30), + Files: Array.Empty()); await stateStore.UpsertAsync(existingRecord, CancellationToken.None); var stateManager = new ExportStateManager(stateStore, timeProvider); @@ -350,6 +352,7 @@ public sealed class TrivyDbFeedExporterTests : IDisposable Assert.Null(updated.LastDeltaDigest); Assert.NotEqual("sha256:old", updated.ExportCursor); Assert.Equal("registry.example/trivy", updated.TargetRepository); + Assert.NotEmpty(updated.Files); } private static Advisory CreateSampleAdvisory( diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md b/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md index dca12c7c..55768757 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md @@ -10,4 +10,4 @@ |End-to-end tests with small dataset|QA|Exporters|DONE – added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.| |ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE – baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.| |Streamed package building to avoid large copies|BE-Export|Exporters|DONE – metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.| -|Plan incremental/delta exports|BE-Export|Exporters|TODO – design reuse of existing blobs/layers when inputs unchanged instead of rewriting full trees each run.| +|Plan incremental/delta exports|BE-Export|Exporters|DOING – export state now persists per-file manifests; planner detects changes/removed files and schedules delta vs full runs, groundwork laid for layer reuse.| diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs index 02ee9cad..f75a2644 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs @@ -1,8 +1,14 @@ namespace StellaOps.Feedser.Exporter.TrivyDb; +using System.Collections.Generic; +using StellaOps.Feedser.Storage.Mongo.Exporting; + public sealed record TrivyDbExportPlan( TrivyDbExportMode Mode, string TreeDigest, string? BaseExportId, string? BaseManifestDigest, - bool ResetBaseline); + bool ResetBaseline, + IReadOnlyList Manifest, + IReadOnlyList ChangedFiles, + IReadOnlyList RemovedPaths); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs index 330902de..0d4d9727 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs @@ -3,40 +3,100 @@ using StellaOps.Feedser.Storage.Mongo.Exporting; namespace StellaOps.Feedser.Exporter.TrivyDb; +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Storage.Mongo.Exporting; + public sealed class TrivyDbExportPlanner { - public TrivyDbExportPlan CreatePlan(ExportStateRecord? existingState, string treeDigest) + public TrivyDbExportPlan CreatePlan( + ExportStateRecord? existingState, + string treeDigest, + IReadOnlyList manifest) { ArgumentException.ThrowIfNullOrEmpty(treeDigest); + manifest ??= Array.Empty(); - if (existingState is null) + if (existingState is null || (existingState.Files?.Count ?? 0) == 0) { return new TrivyDbExportPlan( TrivyDbExportMode.Full, treeDigest, - BaseExportId: null, - BaseManifestDigest: null, - ResetBaseline: true); + BaseExportId: existingState?.BaseExportId, + BaseManifestDigest: existingState?.LastFullDigest, + ResetBaseline: true, + Manifest: manifest, + ChangedFiles: manifest, + RemovedPaths: Array.Empty()); } - if (string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal)) + var existingFiles = existingState.Files ?? Array.Empty(); + var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal); + if (cursorMatches) { return new TrivyDbExportPlan( TrivyDbExportMode.Skip, treeDigest, existingState.BaseExportId, existingState.LastFullDigest, - ResetBaseline: false); + ResetBaseline: false, + Manifest: existingFiles, + ChangedFiles: Array.Empty(), + RemovedPaths: Array.Empty()); + } + + var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); + var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); + + var removed = existingMap.Keys + .Where(path => !newMap.ContainsKey(path)) + .ToArray(); + + if (removed.Length > 0) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: true, + Manifest: manifest, + ChangedFiles: manifest, + RemovedPaths: removed); + } + + var changed = new List(); + foreach (var file in manifest) + { + if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal)) + { + changed.Add(file); + } + } + + if (changed.Count == 0) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Skip, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: false, + Manifest: existingFiles, + ChangedFiles: Array.Empty(), + RemovedPaths: Array.Empty()); } var resetBaseline = existingState.LastDeltaDigest is not null; - - // Placeholder for future delta support – current behavior always rebuilds when tree changes. return new TrivyDbExportPlan( - TrivyDbExportMode.Full, + TrivyDbExportMode.Delta, treeDigest, existingState.BaseExportId, existingState.LastFullDigest, - resetBaseline); + resetBaseline, + Manifest: manifest, + ChangedFiles: changed, + RemovedPaths: Array.Empty()); } } diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs index 9b0258d9..4b8f679f 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs @@ -85,9 +85,13 @@ public sealed class TrivyDbFeedExporter : IFeedExporter jsonResult.AdvisoryCount, jsonResult.TotalBytes); + var manifest = jsonResult.Files + .Select(static file => new ExportFileRecord(file.RelativePath, file.Length, file.Digest)) + .ToArray(); + var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult); var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); - var plan = _exportPlanner.CreatePlan(existingState, treeDigest); + var plan = _exportPlanner.CreatePlan(existingState, treeDigest, manifest); if (plan.Mode == TrivyDbExportMode.Skip) { @@ -104,6 +108,14 @@ public sealed class TrivyDbFeedExporter : IFeedExporter return; } + if (plan.Mode == TrivyDbExportMode.Delta) + { + _logger.LogInformation( + "Trivy DB export {ExportId} identified {ChangedCount} changed JSON files.", + exportId, + plan.ChangedFiles.Count); + } + var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false); var metadataBytes = CreateMetadataJson(builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt); @@ -142,15 +154,29 @@ public sealed class TrivyDbFeedExporter : IFeedExporter resetBaseline = true; } - await _stateManager.StoreFullExportAsync( - ExporterId, - exportId, - ociResult.ManifestDigest, - cursor: treeDigest, - targetRepository: _options.TargetRepository, - exporterVersion: _exporterVersion, - resetBaseline: resetBaseline, - cancellationToken: cancellationToken).ConfigureAwait(false); + if (plan.Mode == TrivyDbExportMode.Full || resetBaseline) + { + await _stateManager.StoreFullExportAsync( + ExporterId, + exportId, + ociResult.ManifestDigest, + cursor: treeDigest, + targetRepository: _options.TargetRepository, + exporterVersion: _exporterVersion, + resetBaseline: resetBaseline, + manifest: plan.Manifest, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await _stateManager.StoreDeltaExportAsync( + ExporterId, + deltaDigest: treeDigest, + cursor: treeDigest, + exporterVersion: _exporterVersion, + manifest: plan.Manifest, + cancellationToken: cancellationToken).ConfigureAwait(false); + } await CreateOfflineBundleAsync(destination, exportId, exportedAt, cancellationToken).ConfigureAwait(false); } diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs b/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs index 126676c9..8bad42ac 100644 --- a/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs +++ b/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs @@ -16,6 +16,7 @@ public sealed class AdvisoryPrecedenceMergerTests { var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero)); var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + using var metrics = new MetricCollector("StellaOps.Feedser.Merge"); var (redHat, nvd) = CreateVendorAndRegistryAdvisories(); var expectedMergeTimestamp = timeProvider.GetUtcNow(); @@ -46,6 +47,14 @@ public sealed class AdvisoryPrecedenceMergerTests Assert.Equal(expectedMergeTimestamp, mergeProvenance.RecordedAt); Assert.Contains("redhat", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); Assert.Contains("nvd", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + + var rangeMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.range_overrides"); + Assert.Equal(1, rangeMeasurement.Value); + Assert.Contains(rangeMeasurement.Tags, tag => string.Equals(tag.Key, "suppressed_source", StringComparison.Ordinal) && tag.Value?.ToString()?.Contains("nvd", StringComparison.OrdinalIgnoreCase) == true); + + var severityConflict = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.conflicts"); + Assert.Equal(1, severityConflict.Value); + Assert.Contains(severityConflict.Tags, tag => string.Equals(tag.Key, "type", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "severity", StringComparison.OrdinalIgnoreCase)); } [Fact] @@ -155,6 +164,13 @@ public sealed class AdvisoryPrecedenceMergerTests Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "primary_source" && string.Equals(tag.Value?.ToString(), "nvd", StringComparison.OrdinalIgnoreCase)); Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "suppressed_source" && tag.Value?.ToString()?.Contains("redhat", StringComparison.OrdinalIgnoreCase) == true); + Assert.DoesNotContain(metrics.Measurements, measurement => measurement.Name == "feedser.merge.range_overrides"); + + var conflictMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.conflicts"); + Assert.Equal(1, conflictMeasurement.Value); + Assert.Contains(conflictMeasurement.Tags, tag => tag.Key == "type" && string.Equals(tag.Value?.ToString(), "severity", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(conflictMeasurement.Tags, tag => tag.Key == "reason" && string.Equals(tag.Value?.ToString(), "mismatch", StringComparison.OrdinalIgnoreCase)); + var logEntry = Assert.Single(logger.Entries, entry => entry.EventId.Name == "AdvisoryOverride"); Assert.Equal(LogLevel.Information, logEntry.Level); Assert.NotNull(logEntry.StructuredState); diff --git a/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs index db2467c9..caca5c46 100644 --- a/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs +++ b/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs @@ -45,14 +45,21 @@ public sealed class AffectedPackagePrecedenceResolverTests }); var resolver = new AffectedPackagePrecedenceResolver(); - var merged = resolver.Merge(new[] { nvd, redHat }); + var result = resolver.Merge(new[] { nvd, redHat }); - var package = Assert.Single(merged); + var package = Assert.Single(result.Packages); Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", package.Identifier); Assert.Empty(package.VersionRanges); // NVD range overridden Assert.Contains(package.Statuses, status => status.Status == "known_affected"); Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); + + var rangeOverride = Assert.Single(result.Overrides); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", rangeOverride.Identifier); + Assert.Equal(0, rangeOverride.PrimaryRank); + Assert.True(rangeOverride.SuppressedRank >= rangeOverride.PrimaryRank); + Assert.Equal(0, rangeOverride.PrimaryRangeCount); + Assert.Equal(1, rangeOverride.SuppressedRangeCount); } [Fact] @@ -78,11 +85,12 @@ public sealed class AffectedPackagePrecedenceResolverTests }); var resolver = new AffectedPackagePrecedenceResolver(); - var merged = resolver.Merge(new[] { nvd }); + var result = resolver.Merge(new[] { nvd }); - var package = Assert.Single(merged); + var package = Assert.Single(result.Packages); Assert.Equal(nvd.Identifier, package.Identifier); Assert.Equal(nvd.VersionRanges.Single().RangeExpression, package.VersionRanges.Single().RangeExpression); Assert.Equal("nvd", package.Provenance.Single().Source); + Assert.Empty(result.Overrides); } } diff --git a/src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs new file mode 100644 index 00000000..e2981df1 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs @@ -0,0 +1,135 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Merge.Tests; + +[Collection("mongo-fixture")] +public sealed class AliasGraphResolverTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public AliasGraphResolverTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task ResolveAsync_ReturnsCollisions_WhenAliasesOverlap() + { + await DropAliasCollectionAsync(); + + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var resolver = new AliasGraphResolver(aliasStore); + + var timestamp = DateTimeOffset.UtcNow; + await aliasStore.ReplaceAsync( + "ADV-1", + new[] { new AliasEntry("CVE", "CVE-2025-2000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-1") }, + timestamp, + CancellationToken.None); + + await aliasStore.ReplaceAsync( + "ADV-2", + new[] { new AliasEntry("CVE", "CVE-2025-2000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-2") }, + timestamp.AddMinutes(1), + CancellationToken.None); + + var result = await resolver.ResolveAsync("ADV-1", CancellationToken.None); + Assert.NotNull(result); + Assert.Equal("ADV-1", result.AdvisoryKey); + Assert.NotEmpty(result.Collisions); + var collision = Assert.Single(result.Collisions); + Assert.Equal("CVE", collision.Scheme); + Assert.Contains("ADV-1", collision.AdvisoryKeys); + Assert.Contains("ADV-2", collision.AdvisoryKeys); + } + + [Fact] + public async Task BuildComponentAsync_TracesConnectedAdvisories() + { + await DropAliasCollectionAsync(); + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var resolver = new AliasGraphResolver(aliasStore); + + var timestamp = DateTimeOffset.UtcNow; + await aliasStore.ReplaceAsync( + "ADV-A", + new[] { new AliasEntry("CVE", "CVE-2025-4000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-A") }, + timestamp, + CancellationToken.None); + + await aliasStore.ReplaceAsync( + "ADV-B", + new[] { new AliasEntry("CVE", "CVE-2025-4000"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-B"), new AliasEntry("OSV", "OSV-2025-1") }, + timestamp.AddMinutes(1), + CancellationToken.None); + + await aliasStore.ReplaceAsync( + "ADV-C", + new[] { new AliasEntry("OSV", "OSV-2025-1"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-C") }, + timestamp.AddMinutes(2), + CancellationToken.None); + + var component = await resolver.BuildComponentAsync("ADV-A", CancellationToken.None); + Assert.Contains("ADV-A", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); + Assert.Contains("ADV-B", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); + Assert.Contains("ADV-C", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); + Assert.NotEmpty(component.Collisions); + Assert.True(component.AliasMap.ContainsKey("ADV-A")); + Assert.Contains(component.AliasMap["ADV-B"], record => record.Scheme == "OSV" && record.Value == "OSV-2025-1"); + } + + private async Task DropAliasCollectionAsync() + { + try + { + await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.Alias); + } + catch (MongoDB.Driver.MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + } + } + + [Fact] + public async Task BuildComponentAsync_LinksOsvAndGhsaAliases() + { + await DropAliasCollectionAsync(); + + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var resolver = new AliasGraphResolver(aliasStore); + var timestamp = DateTimeOffset.UtcNow; + + await aliasStore.ReplaceAsync( + "ADV-OSV", + new[] + { + new AliasEntry("OSV", "OSV-2025-2001"), + new AliasEntry("GHSA", "GHSA-zzzz-zzzz-zzzz"), + new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-OSV"), + }, + timestamp, + CancellationToken.None); + + await aliasStore.ReplaceAsync( + "ADV-GHSA", + new[] + { + new AliasEntry("GHSA", "GHSA-zzzz-zzzz-zzzz"), + new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-GHSA"), + }, + timestamp.AddMinutes(1), + CancellationToken.None); + + var component = await resolver.BuildComponentAsync("ADV-OSV", CancellationToken.None); + + Assert.Contains("ADV-GHSA", component.AdvisoryKeys, StringComparer.OrdinalIgnoreCase); + Assert.Contains(component.Collisions, collision => collision.Scheme == "GHSA" && collision.Value == "GHSA-zzzz-zzzz-zzzz"); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs b/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs index 5297ec41..22283412 100644 --- a/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs +++ b/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs @@ -76,6 +76,26 @@ public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime Assert.True(persisted.BeforeHash.Length > 0); } + [Fact] + public async Task MergePipeline_IsDeterministicAcrossRuns() + { + await EnsureInitializedAsync(); + + var merger = _merger!; + var calculator = new CanonicalHashCalculator(); + + var first = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); + var second = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); + + var firstHash = calculator.ComputeHash(first); + var secondHash = calculator.ComputeHash(second); + + Assert.Equal(firstHash, secondHash); + Assert.Equal(first.AdvisoryKey, second.AdvisoryKey); + Assert.Equal(first.Aliases.Length, second.Aliases.Length); + Assert.True(first.Aliases.SequenceEqual(second.Aliases)); + } + public async Task InitializeAsync() { _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero)) diff --git a/src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs b/src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs new file mode 100644 index 00000000..966d5b12 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Feedser.Merge.Jobs; + +internal static class MergeJobKinds +{ + public const string Reconcile = "merge:reconcile"; +} diff --git a/src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs b/src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs new file mode 100644 index 00000000..4e5004a4 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs @@ -0,0 +1,43 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Merge.Services; + +namespace StellaOps.Feedser.Merge.Jobs; + +public sealed class MergeReconcileJob : IJob +{ + private readonly AdvisoryMergeService _mergeService; + private readonly ILogger _logger; + + public MergeReconcileJob(AdvisoryMergeService mergeService, ILogger logger) + { + _mergeService = mergeService ?? throw new ArgumentNullException(nameof(mergeService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + if (!context.Parameters.TryGetValue("seed", out var seedValue) || seedValue is not string seed || string.IsNullOrWhiteSpace(seed)) + { + context.Logger.LogWarning("merge:reconcile job requires a non-empty 'seed' parameter."); + return; + } + + var result = await _mergeService.MergeAsync(seed, cancellationToken).ConfigureAwait(false); + if (result.Merged is null) + { + _logger.LogInformation("No advisories available to merge for alias component seeded by {Seed}", seed); + return; + } + + _logger.LogInformation( + "Merged alias component seeded by {Seed} into canonical {Canonical} using {Count} advisories; collisions={Collisions}", + seed, + result.CanonicalAdvisoryKey, + result.Inputs.Count, + result.Component.Collisions.Count); + } +} diff --git a/src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs new file mode 100644 index 00000000..acdf31cd --- /dev/null +++ b/src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs @@ -0,0 +1,41 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Merge.Jobs; +using StellaOps.Feedser.Merge.Options; +using StellaOps.Feedser.Merge.Services; + +namespace StellaOps.Feedser.Merge; + +public static class MergeServiceCollectionExtensions +{ + public static IServiceCollection AddMergeModule(this IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(sp => + { + var options = configuration.GetSection("feedser:merge:precedence").Get(); + return options is null ? new AffectedPackagePrecedenceResolver() : new AffectedPackagePrecedenceResolver(options); + }); + + services.TryAddSingleton(sp => + { + var resolver = sp.GetRequiredService(); + var options = configuration.GetSection("feedser:merge:precedence").Get(); + var timeProvider = sp.GetRequiredService(); + var logger = sp.GetRequiredService>(); + return new AdvisoryPrecedenceMerger(resolver, options, timeProvider, logger); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs new file mode 100644 index 00000000..b34aaf30 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Merge.Options; + +/// +/// Provides the built-in precedence table used by the merge engine when no overrides are supplied. +/// +internal static class AdvisoryPrecedenceDefaults +{ + public static IReadOnlyDictionary Rankings { get; } = CreateDefaultTable(); + + private static IReadOnlyDictionary CreateDefaultTable() + { + var table = new Dictionary(StringComparer.OrdinalIgnoreCase); + + // 0 – distro PSIRTs/OVAL feeds (authoritative for OS packages). + Add(table, 0, + "redhat", + "ubuntu", + "distro-ubuntu", + "debian", + "distro-debian", + "suse", + "distro-suse"); + + // 1 – vendor PSIRTs (authoritative product advisories). + Add(table, 1, + "msrc", + "vndr-msrc", + "vndr-oracle", + "vndr_oracle", + "oracle", + "vndr-adobe", + "adobe", + "vndr-apple", + "apple", + "vndr-cisco", + "cisco", + "vmware", + "vndr-vmware", + "vndr_vmware", + "vndr-chromium", + "chromium", + "vendor"); + + // 2 – ecosystem registries (OSS package maintainers). + Add(table, 2, + "ghsa", + "osv", + "cve"); + + // 3 – regional CERT / ICS enrichment feeds. + Add(table, 3, + "jvn", + "acsc", + "cccs", + "cert-fr", + "certfr", + "cert-in", + "certin", + "cert-cc", + "certcc", + "certbund", + "cert-bund", + "ru-bdu", + "ru-nkcki", + "kisa", + "ics-cisa", + "ics-kaspersky"); + + // 4 – KEV / exploit catalogue annotations (flag only). + Add(table, 4, + "kev", + "cisa-kev"); + + // 5 – public registries (baseline data). + Add(table, 5, + "nvd"); + + return table; + } + + private static void Add(IDictionary table, int rank, params string[] sources) + { + foreach (var source in sources) + { + if (string.IsNullOrWhiteSpace(source)) + { + continue; + } + + table[source] = rank; + } + } +} diff --git a/src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs b/src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs new file mode 100644 index 00000000..e5b18741 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs @@ -0,0 +1,190 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +namespace StellaOps.Feedser.Merge.Services; + +public sealed class AdvisoryMergeService +{ + private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); + private static readonly Counter AliasCollisionCounter = MergeMeter.CreateCounter( + "feedser.merge.identity_conflicts", + unit: "count", + description: "Number of alias collisions detected during merge."); + + private static readonly string[] PreferredAliasSchemes = + { + AliasSchemes.Cve, + AliasSchemes.Ghsa, + AliasSchemes.OsV, + AliasSchemes.Msrc, + }; + + private readonly AliasGraphResolver _aliasResolver; + private readonly IAdvisoryStore _advisoryStore; + private readonly AdvisoryPrecedenceMerger _precedenceMerger; + private readonly MergeEventWriter _mergeEventWriter; + private readonly ILogger _logger; + + public AdvisoryMergeService( + AliasGraphResolver aliasResolver, + IAdvisoryStore advisoryStore, + AdvisoryPrecedenceMerger precedenceMerger, + MergeEventWriter mergeEventWriter, + ILogger logger) + { + _aliasResolver = aliasResolver ?? throw new ArgumentNullException(nameof(aliasResolver)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _precedenceMerger = precedenceMerger ?? throw new ArgumentNullException(nameof(precedenceMerger)); + _mergeEventWriter = mergeEventWriter ?? throw new ArgumentNullException(nameof(mergeEventWriter)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task MergeAsync(string seedAdvisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(seedAdvisoryKey); + + var component = await _aliasResolver.BuildComponentAsync(seedAdvisoryKey, cancellationToken).ConfigureAwait(false); + var inputs = new List(); + + foreach (var advisoryKey in component.AdvisoryKeys) + { + cancellationToken.ThrowIfCancellationRequested(); + var advisory = await _advisoryStore.FindAsync(advisoryKey, cancellationToken).ConfigureAwait(false); + if (advisory is not null) + { + inputs.Add(advisory); + } + } + + if (inputs.Count == 0) + { + _logger.LogWarning("Alias component seeded by {Seed} contains no persisted advisories", seedAdvisoryKey); + return AdvisoryMergeResult.Empty(seedAdvisoryKey, component); + } + + var canonicalKey = SelectCanonicalKey(component) ?? seedAdvisoryKey; + var before = await _advisoryStore.FindAsync(canonicalKey, cancellationToken).ConfigureAwait(false); + var normalizedInputs = NormalizeInputs(inputs, canonicalKey); + + Advisory? merged; + try + { + merged = _precedenceMerger.Merge(normalizedInputs); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to merge alias component seeded by {Seed}", seedAdvisoryKey); + throw; + } + + if (component.Collisions.Count > 0) + { + foreach (var collision in component.Collisions) + { + var tags = new KeyValuePair[] + { + new("scheme", collision.Scheme ?? string.Empty), + new("alias_value", collision.Value ?? string.Empty), + new("advisory_count", collision.AdvisoryKeys.Count), + }; + + AliasCollisionCounter.Add(1, tags); + + _logger.LogInformation( + "Alias collision {Scheme}:{Value} involves advisories {Advisories}", + collision.Scheme, + collision.Value, + string.Join(", ", collision.AdvisoryKeys)); + } + } + + if (merged is not null) + { + await _advisoryStore.UpsertAsync(merged, cancellationToken).ConfigureAwait(false); + await _mergeEventWriter.AppendAsync( + canonicalKey, + before, + merged, + Array.Empty(), + cancellationToken).ConfigureAwait(false); + } + + return new AdvisoryMergeResult(seedAdvisoryKey, canonicalKey, component, inputs, before, merged); + } + + private static IEnumerable NormalizeInputs(IEnumerable advisories, string canonicalKey) + { + foreach (var advisory in advisories) + { + yield return CloneWithKey(advisory, canonicalKey); + } + } + + private static Advisory CloneWithKey(Advisory source, string advisoryKey) + => new( + advisoryKey, + source.Title, + source.Summary, + source.Language, + source.Published, + source.Modified, + source.Severity, + source.ExploitKnown, + source.Aliases, + source.References, + source.AffectedPackages, + source.CvssMetrics, + source.Provenance); + + private static string? SelectCanonicalKey(AliasComponent component) + { + foreach (var scheme in PreferredAliasSchemes) + { + var alias = component.AliasMap.Values + .SelectMany(static aliases => aliases) + .FirstOrDefault(record => string.Equals(record.Scheme, scheme, StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrWhiteSpace(alias?.Value)) + { + return alias.Value; + } + } + + if (component.AliasMap.TryGetValue(component.SeedAdvisoryKey, out var seedAliases)) + { + var primary = seedAliases.FirstOrDefault(record => string.Equals(record.Scheme, AliasStoreConstants.PrimaryScheme, StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrWhiteSpace(primary?.Value)) + { + return primary.Value; + } + } + + var firstAlias = component.AliasMap.Values.SelectMany(static aliases => aliases).FirstOrDefault(); + if (!string.IsNullOrWhiteSpace(firstAlias?.Value)) + { + return firstAlias.Value; + } + + return component.SeedAdvisoryKey; + } +} + +public sealed record AdvisoryMergeResult( + string SeedAdvisoryKey, + string CanonicalAdvisoryKey, + AliasComponent Component, + IReadOnlyList Inputs, + Advisory? Previous, + Advisory? Merged) +{ + public static AdvisoryMergeResult Empty(string seed, AliasComponent component) + => new(seed, seed, component, Array.Empty(), null, null); +} diff --git a/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs b/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs index 348a4825..63a1634d 100644 --- a/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs +++ b/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.Metrics; +using System.Globalization; using System.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -14,36 +15,42 @@ namespace StellaOps.Feedser.Merge.Services; /// public sealed class AdvisoryPrecedenceMerger { - private static readonly IReadOnlyDictionary DefaultPrecedence = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["redhat"] = 0, - ["ubuntu"] = 0, - ["debian"] = 0, - ["suse"] = 0, - ["msrc"] = 1, - ["oracle"] = 1, - ["adobe"] = 1, - ["chromium"] = 1, - ["vendor"] = 1, - ["jvn"] = 2, - ["certfr"] = 2, - ["certin"] = 2, - ["ics-kaspersky"] = 2, - ["kev"] = 6, - ["nvd"] = 5, - }; - private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); + private static readonly Counter MergeCounter = MergeMeter.CreateCounter( + "feedser.merge.operations", + unit: "count", + description: "Number of merge invocations executed by the precedence engine."); + private static readonly Counter OverridesCounter = MergeMeter.CreateCounter( "feedser.merge.overrides", unit: "count", description: "Number of times lower-precedence advisories were overridden by higher-precedence sources."); + private static readonly Counter RangeOverrideCounter = MergeMeter.CreateCounter( + "feedser.merge.range_overrides", + unit: "count", + description: "Number of affected-package range overrides performed during precedence merge."); + + private static readonly Counter ConflictCounter = MergeMeter.CreateCounter( + "feedser.merge.conflicts", + unit: "count", + description: "Number of precedence conflicts detected (severity, rank ties, etc.)."); + private static readonly Action OverrideLogged = LoggerMessage.Define( LogLevel.Information, new EventId(1000, "AdvisoryOverride"), "Advisory precedence override {@Override}"); + private static readonly Action RangeOverrideLogged = LoggerMessage.Define( + LogLevel.Information, + new EventId(1001, "PackageRangeOverride"), + "Affected package precedence override {@Override}"); + + private static readonly Action ConflictLogged = LoggerMessage.Define( + LogLevel.Information, + new EventId(1002, "PrecedenceConflict"), + "Precedence conflict {@Conflict}"); + private readonly AffectedPackagePrecedenceResolver _packageResolver; private readonly IReadOnlyDictionary _precedence; private readonly int _fallbackRank; @@ -51,12 +58,12 @@ public sealed class AdvisoryPrecedenceMerger private readonly ILogger _logger; public AdvisoryPrecedenceMerger() - : this(new AffectedPackagePrecedenceResolver(), DefaultPrecedence, System.TimeProvider.System, NullLogger.Instance) + : this(new AffectedPackagePrecedenceResolver(), TimeProvider.System) { } public AdvisoryPrecedenceMerger(AffectedPackagePrecedenceResolver packageResolver, System.TimeProvider? timeProvider = null) - : this(packageResolver, DefaultPrecedence, timeProvider ?? System.TimeProvider.System, NullLogger.Instance) + : this(packageResolver, packageResolver?.Precedence ?? AdvisoryPrecedenceDefaults.Rankings, timeProvider ?? TimeProvider.System, NullLogger.Instance) { } @@ -119,6 +126,8 @@ public sealed class AdvisoryPrecedenceMerger .ThenByDescending(entry => entry.Advisory.Provenance.Length) .ToArray(); + MergeCounter.Add(1, new KeyValuePair("inputs", list.Count)); + var primary = ordered[0].Advisory; var title = PickString(ordered, advisory => advisory.Title) ?? advisoryKey; @@ -137,7 +146,8 @@ public sealed class AdvisoryPrecedenceMerger .Distinct() .ToArray(); - var affectedPackages = _packageResolver.Merge(ordered.SelectMany(entry => entry.Advisory.AffectedPackages)); + var packageResult = _packageResolver.Merge(ordered.SelectMany(entry => entry.Advisory.AffectedPackages)); + var affectedPackages = packageResult.Packages; var cvssMetrics = ordered .SelectMany(entry => entry.Advisory.CvssMetrics) .Distinct() @@ -168,6 +178,8 @@ public sealed class AdvisoryPrecedenceMerger var exploitKnown = ordered.Any(entry => entry.Advisory.ExploitKnown); LogOverrides(advisoryKey, ordered); + LogPackageOverrides(advisoryKey, packageResult.Overrides); + RecordFieldConflicts(advisoryKey, ordered); return new Advisory( advisoryKey, @@ -275,6 +287,125 @@ public sealed class AdvisoryPrecedenceMerger } } + private void LogPackageOverrides(string advisoryKey, IReadOnlyList overrides) + { + if (overrides.Count == 0) + { + return; + } + + foreach (var record in overrides) + { + var tags = new KeyValuePair[] + { + new("advisory_key", advisoryKey), + new("package_type", record.Type), + new("primary_source", FormatSourceLabel(record.PrimarySources)), + new("suppressed_source", FormatSourceLabel(record.SuppressedSources)), + new("primary_rank", record.PrimaryRank), + new("suppressed_rank", record.SuppressedRank), + new("primary_range_count", record.PrimaryRangeCount), + new("suppressed_range_count", record.SuppressedRangeCount), + }; + + RangeOverrideCounter.Add(1, tags); + + var audit = new PackageOverrideAudit( + advisoryKey, + record.Type, + record.Identifier, + record.Platform, + record.PrimaryRank, + record.SuppressedRank, + record.PrimarySources, + record.SuppressedSources, + record.PrimaryRangeCount, + record.SuppressedRangeCount); + + RangeOverrideLogged(_logger, audit, null); + } + } + + private void RecordFieldConflicts(string advisoryKey, IReadOnlyList ordered) + { + if (ordered.Count <= 1) + { + return; + } + + var primary = ordered[0]; + var primarySeverity = NormalizeSeverity(primary.Advisory.Severity); + + for (var i = 1; i < ordered.Count; i++) + { + var candidate = ordered[i]; + var candidateSeverity = NormalizeSeverity(candidate.Advisory.Severity); + + if (!string.IsNullOrEmpty(candidateSeverity)) + { + var reason = string.IsNullOrEmpty(primarySeverity) ? "primary_missing" : "mismatch"; + if (string.IsNullOrEmpty(primarySeverity) || !string.Equals(primarySeverity, candidateSeverity, StringComparison.OrdinalIgnoreCase)) + { + RecordConflict( + advisoryKey, + "severity", + reason, + primary, + candidate, + primarySeverity ?? "(none)", + candidateSeverity); + } + } + + if (candidate.Rank == primary.Rank) + { + RecordConflict( + advisoryKey, + "precedence_tie", + "equal_rank", + primary, + candidate, + primary.Rank.ToString(CultureInfo.InvariantCulture), + candidate.Rank.ToString(CultureInfo.InvariantCulture)); + } + } + } + + private void RecordConflict( + string advisoryKey, + string conflictType, + string reason, + AdvisoryEntry primary, + AdvisoryEntry suppressed, + string? primaryValue, + string? suppressedValue) + { + var tags = new KeyValuePair[] + { + new("type", conflictType), + new("reason", reason), + new("primary_source", FormatSourceLabel(primary.Sources)), + new("suppressed_source", FormatSourceLabel(suppressed.Sources)), + new("primary_rank", primary.Rank), + new("suppressed_rank", suppressed.Rank), + }; + + ConflictCounter.Add(1, tags); + + var audit = new MergeFieldConflictAudit( + advisoryKey, + conflictType, + reason, + primary.Sources, + primary.Rank, + suppressed.Sources, + suppressed.Rank, + primaryValue, + suppressedValue); + + ConflictLogged(_logger, audit, null); + } + private readonly record struct AdvisoryEntry(Advisory Advisory, int Rank) { public IReadOnlyCollection Sources { get; } = Advisory.Provenance @@ -284,12 +415,15 @@ public sealed class AdvisoryPrecedenceMerger .ToArray(); } + private static string? NormalizeSeverity(string? severity) + => SeverityNormalization.Normalize(severity); + private static AffectedPackagePrecedenceResolver EnsureResolver( AffectedPackagePrecedenceResolver? resolver, AdvisoryPrecedenceOptions? options, out IReadOnlyDictionary precedence) { - precedence = AdvisoryPrecedenceTable.Merge(DefaultPrecedence, options); + precedence = AdvisoryPrecedenceTable.Merge(AdvisoryPrecedenceDefaults.Rankings, options); if (resolver is null) { @@ -354,4 +488,27 @@ public sealed class AdvisoryPrecedenceMerger int SuppressedAliasCount, int PrimaryProvenanceCount, int SuppressedProvenanceCount); + + private readonly record struct PackageOverrideAudit( + string AdvisoryKey, + string PackageType, + string Identifier, + string? Platform, + int PrimaryRank, + int SuppressedRank, + IReadOnlyCollection PrimarySources, + IReadOnlyCollection SuppressedSources, + int PrimaryRangeCount, + int SuppressedRangeCount); + + private readonly record struct MergeFieldConflictAudit( + string AdvisoryKey, + string ConflictType, + string Reason, + IReadOnlyCollection PrimarySources, + int PrimaryRank, + IReadOnlyCollection SuppressedSources, + int SuppressedRank, + string? PrimaryValue, + string? SuppressedValue); } diff --git a/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs b/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs index 59028d8f..e9a397d4 100644 --- a/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs +++ b/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs @@ -12,30 +12,16 @@ namespace StellaOps.Feedser.Merge.Services; /// public sealed class AffectedPackagePrecedenceResolver { - private static readonly IReadOnlyDictionary DefaultPrecedence = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["redhat"] = 0, - ["ubuntu"] = 0, - ["debian"] = 0, - ["suse"] = 0, - ["msrc"] = 1, - ["oracle"] = 1, - ["adobe"] = 1, - ["chromium"] = 1, - ["vendor"] = 1, - ["nvd"] = 5, - }; - private readonly IReadOnlyDictionary _precedence; private readonly int _fallbackRank; public AffectedPackagePrecedenceResolver() - : this(DefaultPrecedence) + : this(AdvisoryPrecedenceDefaults.Rankings) { } public AffectedPackagePrecedenceResolver(AdvisoryPrecedenceOptions? options) - : this(AdvisoryPrecedenceTable.Merge(DefaultPrecedence, options)) + : this(AdvisoryPrecedenceTable.Merge(AdvisoryPrecedenceDefaults.Rankings, options)) { } @@ -47,7 +33,7 @@ public sealed class AffectedPackagePrecedenceResolver public IReadOnlyDictionary Precedence => _precedence; - public IReadOnlyList Merge(IEnumerable packages) + public AffectedPackagePrecedenceResult Merge(IEnumerable packages) { ArgumentNullException.ThrowIfNull(packages); @@ -56,41 +42,66 @@ public sealed class AffectedPackagePrecedenceResolver .GroupBy(pkg => (pkg.Type, pkg.Identifier, pkg.Platform ?? string.Empty)); var resolved = new List(); + var overrides = new List(); + foreach (var group in grouped) { var ordered = group - .OrderBy(GetPrecedence) - .ThenByDescending(static pkg => pkg.Provenance.Length) - .ThenByDescending(static pkg => pkg.VersionRanges.Length); + .Select(pkg => new PackageEntry(pkg, GetPrecedence(pkg))) + .OrderBy(static entry => entry.Rank) + .ThenByDescending(static entry => entry.Package.Provenance.Length) + .ThenByDescending(static entry => entry.Package.VersionRanges.Length) + .ToList(); - var primary = ordered.First(); + var primary = ordered[0]; var provenance = ordered - .SelectMany(static pkg => pkg.Provenance) + .SelectMany(static entry => entry.Package.Provenance) .Where(static p => p is not null) .Distinct() .ToImmutableArray(); var statuses = ordered - .SelectMany(static pkg => pkg.Statuses) + .SelectMany(static entry => entry.Package.Statuses) .Distinct(AffectedPackageStatusEqualityComparer.Instance) .ToImmutableArray(); + foreach (var candidate in ordered.Skip(1)) + { + if (candidate.Package.VersionRanges.Length == 0) + { + continue; + } + + overrides.Add(new AffectedPackageOverride( + primary.Package.Type, + primary.Package.Identifier, + string.IsNullOrWhiteSpace(primary.Package.Platform) ? null : primary.Package.Platform, + primary.Rank, + candidate.Rank, + ExtractSources(primary.Package), + ExtractSources(candidate.Package), + primary.Package.VersionRanges.Length, + candidate.Package.VersionRanges.Length)); + } + var merged = new AffectedPackage( primary.Type, primary.Identifier, string.IsNullOrWhiteSpace(primary.Platform) ? null : primary.Platform, - primary.VersionRanges, + primary.Package.VersionRanges, statuses, provenance); resolved.Add(merged); } - return resolved + var packagesResult = resolved .OrderBy(static pkg => pkg.Type, StringComparer.Ordinal) .ThenBy(static pkg => pkg.Identifier, StringComparer.Ordinal) .ThenBy(static pkg => pkg.Platform, StringComparer.Ordinal) .ToImmutableArray(); + + return new AffectedPackagePrecedenceResult(packagesResult, overrides.ToImmutableArray()); } private int GetPrecedence(AffectedPackage package) @@ -111,4 +122,42 @@ public sealed class AffectedPackagePrecedenceResolver return bestRank; } + + private static IReadOnlyList ExtractSources(AffectedPackage package) + { + if (package.Provenance.Length == 0) + { + return Array.Empty(); + } + + return package.Provenance + .Select(static p => p.Source) + .Where(static source => !string.IsNullOrWhiteSpace(source)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private readonly record struct PackageEntry(AffectedPackage Package, int Rank) + { + public string Type => Package.Type; + + public string Identifier => Package.Identifier; + + public string? Platform => string.IsNullOrWhiteSpace(Package.Platform) ? null : Package.Platform; + } } + +public sealed record AffectedPackagePrecedenceResult( + IReadOnlyList Packages, + IReadOnlyList Overrides); + +public sealed record AffectedPackageOverride( + string Type, + string Identifier, + string? Platform, + int PrimaryRank, + int SuppressedRank, + IReadOnlyList PrimarySources, + IReadOnlyList SuppressedSources, + int PrimaryRangeCount, + int SuppressedRangeCount); diff --git a/src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs b/src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs new file mode 100644 index 00000000..b633a259 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Storage.Mongo.Aliases; + +namespace StellaOps.Feedser.Merge.Services; + +public sealed class AliasGraphResolver +{ + private readonly IAliasStore _aliasStore; + + public AliasGraphResolver(IAliasStore aliasStore) + { + _aliasStore = aliasStore ?? throw new ArgumentNullException(nameof(aliasStore)); + } + + public async Task ResolveAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + var aliases = await _aliasStore.GetByAdvisoryAsync(advisoryKey, cancellationToken).ConfigureAwait(false); + var collisions = new List(); + + foreach (var alias in aliases) + { + var candidates = await _aliasStore.GetByAliasAsync(alias.Scheme, alias.Value, cancellationToken).ConfigureAwait(false); + var advisoryKeys = candidates + .Select(static candidate => candidate.AdvisoryKey) + .Where(static key => !string.IsNullOrWhiteSpace(key)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (advisoryKeys.Length <= 1) + { + continue; + } + + collisions.Add(new AliasCollision(alias.Scheme, alias.Value, advisoryKeys)); + } + + var unique = new Dictionary(StringComparer.Ordinal); + foreach (var collision in collisions) + { + var key = $"{collision.Scheme}\u0001{collision.Value}"; + if (!unique.ContainsKey(key)) + { + unique[key] = collision; + } + } + + var distinctCollisions = unique.Values.ToArray(); + + return new AliasIdentityResult(advisoryKey, aliases, distinctCollisions); + } + + public async Task BuildComponentAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + + var visited = new HashSet(StringComparer.OrdinalIgnoreCase); + var queue = new Queue(); + var collisionMap = new Dictionary(StringComparer.Ordinal); + + var aliasCache = new Dictionary>(StringComparer.OrdinalIgnoreCase); + queue.Enqueue(advisoryKey); + + while (queue.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var current = queue.Dequeue(); + if (!visited.Add(current)) + { + continue; + } + + var aliases = await GetAliasesAsync(current, cancellationToken, aliasCache).ConfigureAwait(false); + aliasCache[current] = aliases; + foreach (var alias in aliases) + { + var aliasRecords = await GetAdvisoriesForAliasAsync(alias.Scheme, alias.Value, cancellationToken).ConfigureAwait(false); + var advisoryKeys = aliasRecords + .Select(static record => record.AdvisoryKey) + .Where(static key => !string.IsNullOrWhiteSpace(key)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (advisoryKeys.Length <= 1) + { + continue; + } + + foreach (var candidate in advisoryKeys) + { + if (!visited.Contains(candidate)) + { + queue.Enqueue(candidate); + } + } + + var collision = new AliasCollision(alias.Scheme, alias.Value, advisoryKeys); + var key = $"{collision.Scheme}\u0001{collision.Value}"; + collisionMap.TryAdd(key, collision); + } + } + + var aliasMap = new Dictionary>(aliasCache, StringComparer.OrdinalIgnoreCase); + return new AliasComponent(advisoryKey, visited.ToArray(), collisionMap.Values.ToArray(), aliasMap); + } + + private async Task> GetAliasesAsync( + string advisoryKey, + CancellationToken cancellationToken, + IDictionary> cache) + { + if (cache.TryGetValue(advisoryKey, out var cached)) + { + return cached; + } + + var aliases = await _aliasStore.GetByAdvisoryAsync(advisoryKey, cancellationToken).ConfigureAwait(false); + cache[advisoryKey] = aliases; + return aliases; + } + + private Task> GetAdvisoriesForAliasAsync( + string scheme, + string value, + CancellationToken cancellationToken) + => _aliasStore.GetByAliasAsync(scheme, value, cancellationToken); +} + +public sealed record AliasIdentityResult(string AdvisoryKey, IReadOnlyList Aliases, IReadOnlyList Collisions); + +public sealed record AliasComponent( + string SeedAdvisoryKey, + IReadOnlyList AdvisoryKeys, + IReadOnlyList Collisions, + IReadOnlyDictionary> AliasMap); diff --git a/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj b/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj index 0c42770d..2961edbe 100644 --- a/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj +++ b/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj @@ -8,6 +8,7 @@ + diff --git a/src/StellaOps.Feedser.Merge/TASKS.md b/src/StellaOps.Feedser.Merge/TASKS.md index 6969e339..cb2e9871 100644 --- a/src/StellaOps.Feedser.Merge/TASKS.md +++ b/src/StellaOps.Feedser.Merge/TASKS.md @@ -2,12 +2,12 @@ | Task | Owner(s) | Depends on | Notes | |---|---|---|---| |Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|DONE – `AdvisoryIdentityResolver` builds alias-driven clusters with canonical key selection + unit coverage.| -|Precedence policy engine|BE-Merge|Architecture|PSIRT/OVAL > NVD; CERTs enrich; KEV flag.| +|Precedence policy engine|BE-Merge|Architecture|**DONE** – precedence defaults enforced by `AdvisoryPrecedenceMerger`/`AdvisoryPrecedenceDefaults` with distro/PSIRT overriding registry feeds and CERT/KEV enrichers.| |NEVRA comparer plus tests|BE-Merge (Distro WG)|Source.Distro fixtures|DONE – Added Nevra parser/comparer with tilde-aware rpm ordering and unit coverage.| |Debian EVR comparer plus tests|BE-Merge (Distro WG)|Debian fixtures|DONE – DebianEvr comparer mirrors dpkg ordering with tilde/epoch handling and unit coverage.| |SemVer range resolver plus tests|BE-Merge (OSS WG)|OSV/GHSA fixtures|DONE – SemanticVersionRangeResolver covers introduced/fixed/lastAffected semantics with SemVer ordering tests.| |Canonical hash and merge_event writer|BE-Merge|Models, Storage.Mongo|DONE – Hash calculator + MergeEventWriter compute canonical SHA-256 digests and persist merge events.| -|Conflict detection and metrics|BE-Merge|Core|Counters; structured logs; traces.| -|End-to-end determinism test|QA|Merge, key connectors|Same inputs -> same hashes.| +|Conflict detection and metrics|BE-Merge|Core|**DONE** – merge meters emit override/conflict counters and structured audits (`AdvisoryPrecedenceMerger`).| +|End-to-end determinism test|QA|Merge, key connectors|**DONE** – `MergePrecedenceIntegrationTests.MergePipeline_IsDeterministicAcrossRuns` guards determinism.| |Override audit logging|BE-Merge|Observability|DONE – override audits now emit structured logs plus bounded-tag metrics suitable for prod telemetry.| |Configurable precedence table|BE-Merge|Architecture|DONE – precedence options bind via feedser:merge:precedence:ranks with docs/tests covering operator workflow.| diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs b/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs index d8ff25bf..17917920 100644 --- a/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs +++ b/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs @@ -13,7 +13,8 @@ public sealed class CanonicalExamplesTests public void CanonicalExamplesMatchGoldenSnapshots() { Directory.CreateDirectory(FixtureRoot); - var updateGoldens = string.Equals(Environment.GetEnvironmentVariable(UpdateEnvVar), "1", StringComparison.OrdinalIgnoreCase); + var envValue = Environment.GetEnvironmentVariable(UpdateEnvVar); + var updateGoldens = string.Equals(envValue, "1", StringComparison.OrdinalIgnoreCase); var failures = new List(); foreach (var (name, advisory) in CanonicalExampleFactory.GetExamples()) @@ -36,6 +37,8 @@ public sealed class CanonicalExamplesTests var expected = File.ReadAllText(fixturePath).Replace("\r\n", "\n"); if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) { + var actualPath = Path.Combine(FixtureRoot, $"{name}.actual.json"); + File.WriteAllText(actualPath, snapshot); failures.Add($"Fixture mismatch for {name}. Set {UpdateEnvVar}=1 to regenerate."); } } diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs b/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs index 368c6d5a..27a67f12 100644 --- a/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs +++ b/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Text.Json; using StellaOps.Feedser.Models; @@ -62,4 +63,90 @@ public sealed class CanonicalJsonSerializerTests var normalized2 = snap2.Replace("\r\n", "\n"); Assert.Equal(normalized1, normalized2); } + + [Fact] + public void SerializesRangePrimitivesPayload() + { + var recordedAt = new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero); + var provenance = new AdvisoryProvenance("connector-x", "map", "segment-1", recordedAt); + var primitives = new RangePrimitives( + new SemVerPrimitive( + Introduced: "2.0.0", + IntroducedInclusive: true, + Fixed: "2.3.4", + FixedInclusive: false, + LastAffected: "2.3.3", + LastAffectedInclusive: true, + ConstraintExpression: ">=2.0.0 <2.3.4"), + new NevraPrimitive( + Introduced: new NevraComponent("pkg", 0, "2.0.0", "1", "x86_64"), + Fixed: null, + LastAffected: new NevraComponent("pkg", 0, "2.3.3", "3", "x86_64")), + new EvrPrimitive( + Introduced: new EvrComponent(1, "2.0.0", "1"), + Fixed: new EvrComponent(1, "2.3.4", null), + LastAffected: null), + new Dictionary(StringComparer.Ordinal) + { + ["channel"] = "stable", + }); + + var range = new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "2.0.0", + fixedVersion: "2.3.4", + lastAffectedVersion: "2.3.3", + rangeExpression: ">=2.0.0 <2.3.4", + provenance, + primitives); + + var package = new AffectedPackage( + type: "semver", + identifier: "pkg@2.x", + platform: "linux", + versionRanges: new[] { range }, + statuses: Array.Empty(), + provenance: new[] { provenance }); + + var advisory = new Advisory( + advisoryKey: "TEST-PRIM", + title: "Range primitive serialization", + summary: null, + language: null, + published: recordedAt, + modified: recordedAt, + severity: null, + exploitKnown: false, + aliases: Array.Empty(), + references: Array.Empty(), + affectedPackages: new[] { package }, + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + + var json = CanonicalJsonSerializer.Serialize(advisory); + using var document = JsonDocument.Parse(json); + var rangeElement = document.RootElement + .GetProperty("affectedPackages")[0] + .GetProperty("versionRanges")[0]; + + Assert.True(rangeElement.TryGetProperty("primitives", out var primitivesElement)); + + var semver = primitivesElement.GetProperty("semVer"); + Assert.Equal("2.0.0", semver.GetProperty("introduced").GetString()); + Assert.True(semver.GetProperty("introducedInclusive").GetBoolean()); + Assert.Equal("2.3.4", semver.GetProperty("fixed").GetString()); + Assert.False(semver.GetProperty("fixedInclusive").GetBoolean()); + Assert.Equal("2.3.3", semver.GetProperty("lastAffected").GetString()); + + var nevra = primitivesElement.GetProperty("nevra"); + Assert.Equal("pkg", nevra.GetProperty("introduced").GetProperty("name").GetString()); + Assert.Equal(0, nevra.GetProperty("introduced").GetProperty("epoch").GetInt32()); + + var evr = primitivesElement.GetProperty("evr"); + Assert.Equal(1, evr.GetProperty("introduced").GetProperty("epoch").GetInt32()); + Assert.Equal("2.3.4", evr.GetProperty("fixed").GetProperty("upstreamVersion").GetString()); + + var extensions = primitivesElement.GetProperty("vendorExtensions"); + Assert.Equal("stable", extensions.GetProperty("channel").GetString()); + } } diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json b/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json index c3e29659..13412404 100644 --- a/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json +++ b/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json @@ -19,6 +19,7 @@ "fixedVersion": "2.5.1", "introducedVersion": null, "lastAffectedVersion": null, + "primitives": null, "provenance": { "kind": "map", "recordedAt": "2024-03-05T10:00:00+00:00", @@ -32,6 +33,7 @@ "fixedVersion": "3.2.4", "introducedVersion": "3.0.0", "lastAffectedVersion": null, + "primitives": null, "provenance": { "kind": "map", "recordedAt": "2024-03-05T10:00:00+00:00", diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json b/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json index c4d74e04..14f9e4f4 100644 --- a/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json +++ b/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json @@ -29,6 +29,7 @@ "fixedVersion": "1.0.5", "introducedVersion": "1.0", "lastAffectedVersion": null, + "primitives": null, "provenance": { "kind": "map", "recordedAt": "2024-08-01T12:00:00+00:00", diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json b/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json index faabc1ab..8737abd5 100644 --- a/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json +++ b/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json @@ -35,6 +35,7 @@ "fixedVersion": null, "introducedVersion": "0:4.18.0-553.el8", "lastAffectedVersion": null, + "primitives": null, "provenance": { "kind": "map", "recordedAt": "2024-05-11T09:00:00+00:00", diff --git a/src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs b/src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs new file mode 100644 index 00000000..94c8fe00 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class ProvenanceDiagnosticsTests +{ + [Fact] + public void RecordMissing_AddsExpectedTagsAndDeduplicates() + { + ResetState(); + + var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); + using var listener = CreateListener(measurements); + + var baseline = DateTimeOffset.UtcNow; + ProvenanceDiagnostics.RecordMissing("source-A", "range:pkg", baseline); + ProvenanceDiagnostics.RecordMissing("source-A", "range:pkg", baseline.AddMinutes(5)); + ProvenanceDiagnostics.RecordMissing("source-A", "reference:https://example", baseline.AddMinutes(10)); + + listener.Dispose(); + + Assert.Equal(2, measurements.Count); + + var first = measurements[0]; + Assert.Equal(1, first.Value); + Assert.Equal("feedser.provenance.missing", first.Instrument); + Assert.Equal("source-A", first.Tags["source"]); + Assert.Equal("range:pkg", first.Tags["component"]); + Assert.Equal("range", first.Tags["category"]); + Assert.Equal("high", first.Tags["severity"]); + + var second = measurements[1]; + Assert.Equal("feedser.provenance.missing", second.Instrument); + Assert.Equal("reference", second.Tags["category"]); + Assert.Equal("low", second.Tags["severity"]); + } + + [Fact] + public void ReportResumeWindow_ClearsTrackedEntries_WhenWindowBackfills() + { + ResetState(); + + var timestamp = DateTimeOffset.UtcNow; + ProvenanceDiagnostics.RecordMissing("source-B", "package:lib", timestamp); + + var (recorded, earliest, syncRoot) = GetInternalState(); + lock (syncRoot) + { + Assert.True(earliest.ContainsKey("source-B")); + Assert.Contains(recorded, entry => entry.StartsWith("source-B|", StringComparison.OrdinalIgnoreCase)); + } + + ProvenanceDiagnostics.ReportResumeWindow("source-B", timestamp.AddMinutes(-5), NullLogger.Instance); + + lock (syncRoot) + { + Assert.False(earliest.ContainsKey("source-B")); + Assert.DoesNotContain(recorded, entry => entry.StartsWith("source-B|", StringComparison.OrdinalIgnoreCase)); + } + } + + [Fact] + public void ReportResumeWindow_RetainsEntries_WhenWindowTooRecent() + { + ResetState(); + + var timestamp = DateTimeOffset.UtcNow; + ProvenanceDiagnostics.RecordMissing("source-C", "range:pkg", timestamp); + + ProvenanceDiagnostics.ReportResumeWindow("source-C", timestamp.AddMinutes(1), NullLogger.Instance); + + var (recorded, earliest, syncRoot) = GetInternalState(); + lock (syncRoot) + { + Assert.True(earliest.ContainsKey("source-C")); + Assert.Contains(recorded, entry => entry.StartsWith("source-C|", StringComparison.OrdinalIgnoreCase)); + } + } + + [Fact] + public void RecordRangePrimitive_EmitsCoverageMetric() + { + var range = new AffectedVersionRange( + rangeKind: "evr", + introducedVersion: "1:1.1.1n-0+deb11u2", + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: new AdvisoryProvenance("source-D", "range", "pkg", DateTimeOffset.UtcNow), + primitives: new RangePrimitives( + SemVer: null, + Nevra: null, + Evr: new EvrPrimitive( + new EvrComponent(1, "1.1.1n", "0+deb11u2"), + null, + null), + VendorExtensions: new Dictionary { ["debian.release"] = "bullseye" })); + + var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); + using var listener = CreateListener(measurements, "feedser.range.primitives"); + + ProvenanceDiagnostics.RecordRangePrimitive("source-D", range); + + listener.Dispose(); + + var record = Assert.Single(measurements); + Assert.Equal("feedser.range.primitives", record.Instrument); + Assert.Equal(1, record.Value); + Assert.Equal("source-D", record.Tags["source"]); + Assert.Equal("evr", record.Tags["rangeKind"]); + Assert.Equal("evr", record.Tags["primitiveKinds"]); + Assert.Equal("true", record.Tags["hasVendorExtensions"]); + } + + private static MeterListener CreateListener( + List<(string Instrument, long Value, IReadOnlyDictionary Tags)> measurements, + params string[] instrumentNames) + { + var allowed = instrumentNames is { Length: > 0 } ? instrumentNames : new[] { "feedser.provenance.missing" }; + var allowedSet = new HashSet(allowed, StringComparer.OrdinalIgnoreCase); + + var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name == "StellaOps.Feedser.Models.Provenance" && allowedSet.Contains(instrument.Name)) + { + l.EnableMeasurementEvents(instrument); + } + } + }; + + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tag in tags) + { + dict[tag.Key] = tag.Value; + } + + measurements.Add((instrument.Name, measurement, dict)); + }); + + listener.Start(); + return listener; + } + + private static void ResetState() + { + var (_, _, syncRoot) = GetInternalState(); + lock (syncRoot) + { + var (recorded, earliest, _) = GetInternalState(); + recorded.Clear(); + earliest.Clear(); + } + } + + private static (HashSet Recorded, Dictionary Earliest, object SyncRoot) GetInternalState() + { + var type = typeof(ProvenanceDiagnostics); + var recordedField = type.GetField("RecordedComponents", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("RecordedComponents not found"); + var earliestField = type.GetField("EarliestMissing", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("EarliestMissing not found"); + var syncField = type.GetField("SyncRoot", BindingFlags.NonPublic | BindingFlags.Static) ?? throw new InvalidOperationException("SyncRoot not found"); + + var recorded = (HashSet)recordedField.GetValue(null)!; + var earliest = (Dictionary)earliestField.GetValue(null)!; + var sync = syncField.GetValue(null)!; + return (recorded, earliest, sync); + } +} diff --git a/src/StellaOps.Feedser.Models/AffectedVersionRange.cs b/src/StellaOps.Feedser.Models/AffectedVersionRange.cs index 4a211beb..90322308 100644 --- a/src/StellaOps.Feedser.Models/AffectedVersionRange.cs +++ b/src/StellaOps.Feedser.Models/AffectedVersionRange.cs @@ -14,7 +14,8 @@ public sealed record AffectedVersionRange string? fixedVersion, string? lastAffectedVersion, string? rangeExpression, - AdvisoryProvenance provenance) + AdvisoryProvenance provenance, + RangePrimitives? primitives = null) { RangeKind = Validation.EnsureNotNullOrWhiteSpace(rangeKind, nameof(rangeKind)).ToLowerInvariant(); IntroducedVersion = Validation.TrimToNull(introducedVersion); @@ -22,6 +23,7 @@ public sealed record AffectedVersionRange LastAffectedVersion = Validation.TrimToNull(lastAffectedVersion); RangeExpression = Validation.TrimToNull(rangeExpression); Provenance = provenance ?? AdvisoryProvenance.Empty; + Primitives = primitives; } /// @@ -51,6 +53,8 @@ public sealed record AffectedVersionRange public AdvisoryProvenance Provenance { get; } + public RangePrimitives? Primitives { get; } + public string CreateDeterministicKey() => string.Join('|', RangeKind, IntroducedVersion ?? string.Empty, FixedVersion ?? string.Empty, LastAffectedVersion ?? string.Empty, RangeExpression ?? string.Empty); } diff --git a/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md b/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md index 2b9be57e..4e78009d 100644 --- a/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md +++ b/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md @@ -63,6 +63,7 @@ Deterministic ordering: packages sorted by `type`, then `identifier`, then `plat | `lastAffectedVersion` | string? | optional | Inclusive upper bound when no fix exists. | | `rangeExpression` | string? | optional | Normalized textual expression for non-simple ranges. | | `provenance` | AdvisoryProvenance | yes | Provenance entry for the range. | +| `primitives` | RangePrimitives? | optional | Structured metadata (SemVer/Nevra/Evr/vendor extensions) when available. | Comparers/equality ignore provenance differences. diff --git a/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md b/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md index 2536cafa..0e4c1447 100644 --- a/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md +++ b/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md @@ -10,3 +10,5 @@ - **Merge policy**: never discard provenance when merging; instead append a new `AdvisoryProvenance` entry with the merge routine (`source=merge.determine-precedence`). - **Determinism**: provenance collections are sorted by source → kind → recordedAt before serialization; avoid generating random identifiers inside provenance. - **Redaction**: keep provenance values free of secrets; prefer tokens or normalized descriptors when referencing authenticated fetches. +- **Range telemetry**: each `AffectedVersionRange` is observed by the `feedser.range.primitives` metric. Emit the richest `RangePrimitives` possible (SemVer/NEVRA/EVR plus vendor extensions); the telemetry tags make it easy to spot connectors missing structured range data. +- **Vendor extensions**: when vendor feeds surface bespoke status flags, capture them in `RangePrimitives.VendorExtensions`. SUSE advisories publish `suse.status` (open/resolved/investigating) and Ubuntu notices expose `ubuntu.pocket`/`ubuntu.release` to distinguish security vs ESM pockets; Adobe APSB bulletins emit `adobe.track`, `adobe.platform`, `adobe.priority`, `adobe.availability`, plus `adobe.affected.raw`/`adobe.updated.raw` to preserve PSIRT metadata while keeping the status catalog canonical. These values are exported for dashboards and alerting. diff --git a/src/StellaOps.Feedser.Models/ProvenanceInspector.cs b/src/StellaOps.Feedser.Models/ProvenanceInspector.cs new file mode 100644 index 00000000..bcf82642 --- /dev/null +++ b/src/StellaOps.Feedser.Models/ProvenanceInspector.cs @@ -0,0 +1,253 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Models; + +public static class ProvenanceInspector +{ + public static IReadOnlyList FindMissingProvenance(Advisory advisory) + { + var results = new List(); + var source = advisory.Provenance.FirstOrDefault()?.Source ?? "unknown"; + + if (advisory.Provenance.Length == 0) + { + results.Add(new MissingProvenance(source, "advisory", null)); + } + + foreach (var reference in advisory.References) + { + if (IsMissing(reference.Provenance)) + { + results.Add(new MissingProvenance(reference.Provenance.Source ?? source, $"reference:{reference.Url}", reference.Provenance.RecordedAt)); + } + } + + foreach (var package in advisory.AffectedPackages) + { + if (package.Provenance.Length == 0) + { + results.Add(new MissingProvenance(source, $"package:{package.Identifier}", null)); + } + + foreach (var range in package.VersionRanges) + { + ProvenanceDiagnostics.RecordRangePrimitive(range.Provenance.Source ?? source, range); + + if (IsMissing(range.Provenance)) + { + results.Add(new MissingProvenance(range.Provenance.Source ?? source, $"range:{package.Identifier}", range.Provenance.RecordedAt)); + } + } + + foreach (var status in package.Statuses) + { + if (IsMissing(status.Provenance)) + { + results.Add(new MissingProvenance(status.Provenance.Source ?? source, $"status:{package.Identifier}:{status.Status}", status.Provenance.RecordedAt)); + } + } + } + + foreach (var metric in advisory.CvssMetrics) + { + if (IsMissing(metric.Provenance)) + { + results.Add(new MissingProvenance(metric.Provenance.Source ?? source, $"cvss:{metric.Version}", metric.Provenance.RecordedAt)); + } + } + + return results; + } + + private static bool IsMissing(AdvisoryProvenance provenance) + { + return provenance == AdvisoryProvenance.Empty + || string.IsNullOrWhiteSpace(provenance.Source) + || string.IsNullOrWhiteSpace(provenance.Kind); + } +} + +public sealed record MissingProvenance(string Source, string Component, DateTimeOffset? RecordedAt); + +public static class ProvenanceDiagnostics +{ + private static readonly Meter Meter = new("StellaOps.Feedser.Models.Provenance"); + private static readonly Counter MissingCounter = Meter.CreateCounter( + "feedser.provenance.missing", + unit: "count", + description: "Number of canonical objects missing provenance metadata."); + private static readonly Counter RangePrimitiveCounter = Meter.CreateCounter( + "feedser.range.primitives", + unit: "count", + description: "Range coverage by kind, primitive availability, and vendor extensions."); + + private static readonly object SyncRoot = new(); + private static readonly Dictionary EarliestMissing = new(StringComparer.OrdinalIgnoreCase); + private static readonly HashSet RecordedComponents = new(StringComparer.OrdinalIgnoreCase); + + public static void RecordMissing(string source, string component, DateTimeOffset? recordedAt) + { + if (string.IsNullOrWhiteSpace(source)) + { + source = "unknown"; + } + + component = string.IsNullOrWhiteSpace(component) ? "unknown" : component.Trim(); + + bool shouldRecord; + lock (SyncRoot) + { + var key = $"{source}|{component}"; + shouldRecord = RecordedComponents.Add(key); + + if (recordedAt.HasValue) + { + if (!EarliestMissing.TryGetValue(source, out var existing) || recordedAt.Value < existing) + { + EarliestMissing[source] = recordedAt.Value; + } + } + } + + if (!shouldRecord) + { + return; + } + + var category = DetermineCategory(component); + var severity = DetermineSeverity(category); + + var tags = new[] + { + new KeyValuePair("source", source), + new KeyValuePair("component", component), + new KeyValuePair("category", category), + new KeyValuePair("severity", severity), + }; + MissingCounter.Add(1, tags); + } + + public static void ReportResumeWindow(string source, DateTimeOffset windowStart, ILogger logger) + { + if (string.IsNullOrWhiteSpace(source) || logger is null) + { + return; + } + + DateTimeOffset earliest; + var hasEntry = false; + lock (SyncRoot) + { + if (EarliestMissing.TryGetValue(source, out earliest)) + { + hasEntry = true; + if (windowStart <= earliest) + { + EarliestMissing.Remove(source); + var prefix = source + "|"; + RecordedComponents.RemoveWhere(entry => entry.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)); + } + } + } + + if (!hasEntry) + { + return; + } + + if (windowStart <= earliest) + { + logger.LogInformation( + "Resume window starting {WindowStart:o} for {Source} may backfill missing provenance recorded at {Earliest:o}.", + windowStart, + source, + earliest); + } + else + { + logger.LogInformation( + "Earliest missing provenance for {Source} remains at {Earliest:o}; current resume window begins at {WindowStart:o}. Consider widening overlap to backfill.", + source, + earliest, + windowStart); + } + } + + public static void RecordRangePrimitive(string source, AffectedVersionRange range) + { + if (range is null) + { + return; + } + + source = string.IsNullOrWhiteSpace(source) ? "unknown" : source.Trim(); + + var primitives = range.Primitives; + var primitiveKinds = DeterminePrimitiveKinds(primitives); + var vendorExtensions = primitives?.VendorExtensions?.Count ?? 0; + + var tags = new[] + { + new KeyValuePair("source", source), + new KeyValuePair("rangeKind", string.IsNullOrWhiteSpace(range.RangeKind) ? "unknown" : range.RangeKind), + new KeyValuePair("primitiveKinds", primitiveKinds), + new KeyValuePair("hasVendorExtensions", vendorExtensions > 0 ? "true" : "false"), + }; + + RangePrimitiveCounter.Add(1, tags); + } + + private static string DetermineCategory(string component) + { + if (string.IsNullOrWhiteSpace(component)) + { + return "unknown"; + } + + var index = component.IndexOf(':'); + var category = index > 0 ? component[..index] : component; + return category.Trim().ToLowerInvariant(); + } + + private static string DetermineSeverity(string category) + => category switch + { + "advisory" => "critical", + "package" => "high", + "range" => "high", + "status" => "medium", + "cvss" => "medium", + "reference" => "low", + _ => "info", + }; + + private static string DeterminePrimitiveKinds(RangePrimitives? primitives) + { + if (primitives is null) + { + return "none"; + } + + var kinds = new List(3); + if (primitives.SemVer is not null) + { + kinds.Add("semver"); + } + + if (primitives.Nevra is not null) + { + kinds.Add("nevra"); + } + + if (primitives.Evr is not null) + { + kinds.Add("evr"); + } + + return kinds.Count == 0 ? "vendor" : string.Join('+', kinds); + } +} diff --git a/src/StellaOps.Feedser.Models/RangePrimitives.cs b/src/StellaOps.Feedser.Models/RangePrimitives.cs new file mode 100644 index 00000000..b323f8e5 --- /dev/null +++ b/src/StellaOps.Feedser.Models/RangePrimitives.cs @@ -0,0 +1,58 @@ +using System.Collections.Generic; + +namespace StellaOps.Feedser.Models; + +/// +/// Optional structured representations of range semantics attached to . +/// +public sealed record RangePrimitives( + SemVerPrimitive? SemVer, + NevraPrimitive? Nevra, + EvrPrimitive? Evr, + IReadOnlyDictionary? VendorExtensions); + +/// +/// Structured SemVer metadata for a version range. +/// +public sealed record SemVerPrimitive( + string? Introduced, + bool IntroducedInclusive, + string? Fixed, + bool FixedInclusive, + string? LastAffected, + bool LastAffectedInclusive, + string? ConstraintExpression); + +/// +/// Structured NEVRA metadata for a version range. +/// +public sealed record NevraPrimitive( + NevraComponent? Introduced, + NevraComponent? Fixed, + NevraComponent? LastAffected); + +/// +/// Structured Debian EVR metadata for a version range. +/// +public sealed record EvrPrimitive( + EvrComponent? Introduced, + EvrComponent? Fixed, + EvrComponent? LastAffected); + +/// +/// Normalized NEVRA component. +/// +public sealed record NevraComponent( + string Name, + int Epoch, + string Version, + string Release, + string? Architecture); + +/// +/// Normalized EVR component (epoch:upstream revision). +/// +public sealed record EvrComponent( + int Epoch, + string UpstreamVersion, + string? Revision); diff --git a/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj b/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj index ecc3af66..6cd097d3 100644 --- a/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj +++ b/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj @@ -6,4 +6,7 @@ enable true + + + diff --git a/src/StellaOps.Feedser.Models/TASKS.md b/src/StellaOps.Feedser.Models/TASKS.md index bdfe7902..1fb8ef8a 100644 --- a/src/StellaOps.Feedser.Models/TASKS.md +++ b/src/StellaOps.Feedser.Models/TASKS.md @@ -9,8 +9,8 @@ |Docs: field provenance guidelines|BE-Merge|Models|DONE – see `PROVENANCE_GUIDELINES.md`.| |Canonical record definitions kept in sync|BE-Merge|Models|DONE – documented in `CANONICAL_RECORDS.md`; update alongside model changes.| |Alias scheme registry and validation helpers|BE-Merge|Models|DONE – see `AliasSchemes` & `AliasSchemeRegistry` plus validation integration/tests.| -|Range primitives for SemVer/EVR/NEVRA metadata|BE-Merge|Models|TODO – keep structured values without parsing logic; ensure merge/export parity.| -|Provenance envelope field masks|BE-Merge|Models|TODO – guarantee traceability for each mapped field.| +|Range primitives for SemVer/EVR/NEVRA metadata|BE-Merge|Models|DOING – envelope + AdvisoryStore deserialisation landed; VMware/Oracle/Chromium/NVD emit primitives. Remaining connectors (Debian, SUSE, Ubuntu, Apple, Adobe, etc.) still need structured coverage + EVR population.| +|Provenance envelope field masks|BE-Merge|Models|DOING – add richer metric tags (component category/severity), dedupe missing counts, propagate resume logging across connectors.| |Backward-compatibility playbook|BE-Merge, QA|Models|DONE – see `BACKWARD_COMPATIBILITY.md` for evolution policy/test checklist.| |Golden canonical examples|QA|Models|DONE – added `/p:UpdateGoldens=true` test hook wiring `UPDATE_GOLDENS=1` so canonical fixtures regenerate via `dotnet test`; docs/tests unchanged.| |Serialization determinism regression tests|QA|Models|DONE – locale-stability tests hash canonical serializer output across multiple cultures and runs.| diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs index 4323cc1d..a9b8111e 100644 --- a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs @@ -289,8 +289,15 @@ public sealed class CertFrConnectorTests : IAsyncLifetime private static string ReadFixture(string filename) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "CertFr", "Fixtures", filename); - return File.ReadAllText(path); + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "CertFr", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(baseDirectory, "CertFr", "Fixtures", filename); + return File.ReadAllText(fallback); } private static string Normalize(string value) diff --git a/src/StellaOps.Feedser.Source.CertFr/AGENTS.md b/src/StellaOps.Feedser.Source.CertFr/AGENTS.md index 5049da1f..a3160672 100644 --- a/src/StellaOps.Feedser.Source.CertFr/AGENTS.md +++ b/src/StellaOps.Feedser.Source.CertFr/AGENTS.md @@ -19,10 +19,9 @@ ANSSI CERT-FR advisories connector (avis/alertes) providing national enrichment: In: advisory metadata extraction, references, severity text, watermarking. Out: OVAL or package-level authority. ## Observability & security expectations -- Metrics: certfr.fetch.items, certfr.parse.fail, certfr.map.count. +- Metrics: SourceDiagnostics emits shared `feedser.source.http.*` counters/histograms tagged `feedser.source=certfr`, covering fetch counts, parse failures, and map activity. - Logs: feed URL(s), item ids/urls, extraction durations; no PII; allowlist hostnames. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.CertFr.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs index af48a338..6d041382 100644 --- a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs @@ -84,7 +84,8 @@ public sealed class CertInConnectorTests : IAsyncLifetime var normalizedActual = NormalizeLineEndings(canonical); if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) { - var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "CertIn", "Fixtures", "expected-advisory.actual.json"); + var actualPath = ResolveFixturePath("expected-advisory.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); File.WriteAllText(actualPath, canonical); } @@ -316,9 +317,18 @@ public sealed class CertInConnectorTests : IAsyncLifetime => _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); private static string ReadFixture(string filename) + => File.ReadAllText(ResolveFixturePath(filename)); + + private static string ResolveFixturePath(string filename) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "CertIn", "Fixtures", filename); - return File.ReadAllText(path); + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "CertIn", "Fixtures", filename); + if (File.Exists(primary) || filename.EndsWith(".actual.json", StringComparison.OrdinalIgnoreCase)) + { + return primary; + } + + return Path.Combine(baseDirectory, "CertIn", "Fixtures", filename); } private static string NormalizeLineEndings(string value) diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json index 711c20d1..58571cbc 100644 --- a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json @@ -94,4 +94,4 @@ "severity": "high", "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).", "title": "Multiple vulnerabilities in Example Gateway" -} +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertIn/AGENTS.md b/src/StellaOps.Feedser.Source.CertIn/AGENTS.md index ebb7215e..ddd9bdb0 100644 --- a/src/StellaOps.Feedser.Source.CertIn/AGENTS.md +++ b/src/StellaOps.Feedser.Source.CertIn/AGENTS.md @@ -20,10 +20,9 @@ CERT-In national CERT connector; enrichment advisories for India; maps CVE lists In: enrichment, aliasing where stable, references, mitigation text. Out: package range authority; scraping behind auth walls. ## Observability & security expectations -- Metrics: certin.fetch.items, certin.parse.fail, certin.map.enriched_count. +- Metrics: shared `feedser.source.http.*` counters/histograms from SourceDiagnostics tagged `feedser.source=certin` capture fetch volume, parse failures, and map enrich counts. - Logs: advisory codes, CVE counts per advisory, timing; allowlist host; redact personal data if present. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.CertIn.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Common/AGENTS.md b/src/StellaOps.Feedser.Source.Common/AGENTS.md index 723af148..c7e54b50 100644 --- a/src/StellaOps.Feedser.Source.Common/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Common/AGENTS.md @@ -22,11 +22,10 @@ Shared connector toolkit. Provides HTTP clients, retry/backoff, conditional GET In: HTTP plumbing, validators, cursor/backoff utilities, hashing. Out: connector-specific schemas/mapping rules, merge precedence. ## Observability & security expectations -- Metrics: http.req.count, http.retry.count, rate_limit.remaining, validator.fail.count. +- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged with `feedser.source=` plus retries/failures; connector dashboards slice on that tag instead of bespoke metric names. - Logs include uri, status, retries, etag; redact tokens and auth headers. - Distributed tracing hooks and per-connector counters should be wired centrally for consistent observability. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Common.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs new file mode 100644 index 00000000..f670af7e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs @@ -0,0 +1,250 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.Debian.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Distro.Debian.Tests; + +[Collection("mongo-fixture")] +public sealed class DebianConnectorTests : IAsyncLifetime +{ + private static readonly Uri ListUri = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list"); + private static readonly Uri DetailResolved = new("https://security-tracker.debian.org/tracker/DSA-2024-123"); + private static readonly Uri DetailOpen = new("https://security-tracker.debian.org/tracker/DSA-2024-124"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private readonly ITestOutputHelper _output; + + public DebianConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _handler = new CannedHttpMessageHandler(); + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 12, 0, 0, 0, TimeSpan.Zero)); + _output = output; + } + + [Fact] + public async Task FetchParseMap_PopulatesRangePrimitivesAndResumesWithNotModified() + { + await using var provider = await BuildServiceProviderAsync(); + + SeedInitialResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var resolved = advisories.Single(a => a.AdvisoryKey == "DSA-2024-123"); + Assert.Contains("CVE-2024-1000", resolved.Aliases); + Assert.Contains("CVE-2024-1001", resolved.Aliases); + var resolvedBookworm = Assert.Single(resolved.AffectedPackages, p => p.Platform == "bookworm"); + var resolvedRange = Assert.Single(resolvedBookworm.VersionRanges); + Assert.Equal("evr", resolvedRange.RangeKind); + Assert.Equal("1:1.1.1n-0+deb11u2", resolvedRange.IntroducedVersion); + Assert.Equal("1:1.1.1n-0+deb11u5", resolvedRange.FixedVersion); + Assert.NotNull(resolvedRange.Primitives); + Assert.NotNull(resolvedRange.Primitives!.Evr); + Assert.Equal(1, resolvedRange.Primitives.Evr!.Introduced!.Epoch); + Assert.Equal("1.1.1n", resolvedRange.Primitives.Evr.Introduced.UpstreamVersion); + + var open = advisories.Single(a => a.AdvisoryKey == "DSA-2024-124"); + Assert.Contains("CVE-2024-2000", open.Aliases); + var openBookworm = Assert.Single(open.AffectedPackages, p => p.Platform == "bookworm"); + var openRange = Assert.Single(openBookworm.VersionRanges); + Assert.Equal("evr", openRange.RangeKind); + Assert.Equal("1:1.3.1-1", openRange.IntroducedVersion); + Assert.Null(openRange.FixedVersion); + Assert.NotNull(openRange.Primitives); + Assert.NotNull(openRange.Primitives!.Evr); + + // Ensure data persisted through Mongo round-trip. + var found = await advisoryStore.FindAsync("DSA-2024-123", CancellationToken.None); + Assert.NotNull(found); + var persistedRange = Assert.Single(found!.AffectedPackages, pkg => pkg.Platform == "bookworm").VersionRanges.Single(); + Assert.NotNull(persistedRange.Primitives); + Assert.NotNull(persistedRange.Primitives!.Evr); + + // Second run should issue conditional requests and no additional parsing/mapping. + SeedNotModifiedResponses(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documents = provider.GetRequiredService(); + var listDoc = await documents.FindBySourceAndUriAsync(DebianConnectorPlugin.SourceName, DetailResolved.ToString(), CancellationToken.None); + Assert.NotNull(listDoc); + + var refreshed = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, refreshed.Count); + _handler.AssertNoPendingResponses(); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName, CancellationToken.None); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(new TestOutputLoggerProvider(_output))); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddDebianConnector(options => + { + options.ListEndpoint = ListUri; + options.DetailBaseUri = new Uri("https://security-tracker.debian.org/tracker/"); + options.MaxAdvisoriesPerFetch = 10; + options.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(DebianOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + AddListResponse("debian-list.txt", "\"list-v1\""); + AddDetailResponse(DetailResolved, "debian-detail-dsa-2024-123.html", "\"detail-123\""); + AddDetailResponse(DetailOpen, "debian-detail-dsa-2024-124.html", "\"detail-124\""); + } + + private void SeedNotModifiedResponses() + { + AddNotModifiedResponse(ListUri, "\"list-v1\""); + AddNotModifiedResponse(DetailResolved, "\"detail-123\""); + AddNotModifiedResponse(DetailOpen, "\"detail-124\""); + } + + private void AddListResponse(string fixture, string etag) + { + _handler.AddResponse(ListUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/plain"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void AddDetailResponse(Uri uri, string fixture, string etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void AddNotModifiedResponse(Uri uri, string etag) + { + _handler.AddResponse(uri, request => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private static string ReadFixture(string filename) + { + var primary = Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "Debian", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + throw new FileNotFoundException($"Fixture '{filename}' not found", filename); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + private sealed class TestOutputLoggerProvider : ILoggerProvider + { + private readonly ITestOutputHelper _output; + + public TestOutputLoggerProvider(ITestOutputHelper output) => _output = output; + + public ILogger CreateLogger(string categoryName) => new TestOutputLogger(_output); + + public void Dispose() + { + } + + private sealed class TestOutputLogger : ILogger + { + private readonly ITestOutputHelper _output; + + public TestOutputLogger(ITestOutputHelper output) => _output = output; + + public IDisposable BeginScope(TState state) where TState : notnull => NullLogger.Instance.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => false; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (IsEnabled(logLevel)) + { + _output.WriteLine(formatter(state, exception)); + } + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs new file mode 100644 index 00000000..47db9731 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs @@ -0,0 +1,88 @@ +using System; +using Xunit; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Distro.Debian; +using StellaOps.Feedser.Source.Distro.Debian.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Distro.Debian.Tests; + +public sealed class DebianMapperTests +{ + [Fact] + public void Map_BuildsRangePrimitives_ForResolvedPackage() + { + var dto = new DebianAdvisoryDto( + AdvisoryId: "DSA-2024-123", + SourcePackage: "openssl", + Title: "Openssl security update", + Description: "Fixes multiple issues.", + CveIds: new[] { "CVE-2024-1000", "CVE-2024-1001" }, + Packages: new[] + { + new DebianPackageStateDto( + Package: "openssl", + Release: "bullseye", + Status: "resolved", + IntroducedVersion: "1:1.1.1n-0+deb11u2", + FixedVersion: "1:1.1.1n-0+deb11u5", + LastAffectedVersion: null, + Published: new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero)), + new DebianPackageStateDto( + Package: "openssl", + Release: "bookworm", + Status: "open", + IntroducedVersion: null, + FixedVersion: null, + LastAffectedVersion: null, + Published: null) + }, + References: new[] + { + new DebianReferenceDto( + Url: "https://security-tracker.debian.org/tracker/DSA-2024-123", + Kind: "advisory", + Title: "Debian Security Advisory 2024-123"), + }); + + var document = new DocumentRecord( + Id: Guid.NewGuid(), + SourceName: DebianConnectorPlugin.SourceName, + Uri: "https://security-tracker.debian.org/tracker/DSA-2024-123", + FetchedAt: new DateTimeOffset(2024, 9, 1, 1, 0, 0, TimeSpan.Zero), + Sha256: "sha", + Status: "Fetched", + ContentType: "application/json", + Headers: null, + Metadata: null, + Etag: null, + LastModified: null, + GridFsId: null); + + Advisory advisory = DebianMapper.Map(dto, document, new DateTimeOffset(2024, 9, 1, 2, 0, 0, TimeSpan.Zero)); + + Assert.Equal("DSA-2024-123", advisory.AdvisoryKey); + Assert.Contains("CVE-2024-1000", advisory.Aliases); + Assert.Contains("CVE-2024-1001", advisory.Aliases); + + var resolvedPackage = Assert.Single(advisory.AffectedPackages, p => p.Platform == "bullseye"); + var range = Assert.Single(resolvedPackage.VersionRanges); + Assert.Equal("evr", range.RangeKind); + Assert.Equal("1:1.1.1n-0+deb11u2", range.IntroducedVersion); + Assert.Equal("1:1.1.1n-0+deb11u5", range.FixedVersion); + Assert.NotNull(range.Primitives); + var evr = range.Primitives!.Evr; + Assert.NotNull(evr); + Assert.NotNull(evr!.Introduced); + Assert.Equal(1, evr.Introduced!.Epoch); + Assert.Equal("1.1.1n", evr.Introduced.UpstreamVersion); + Assert.Equal("0+deb11u2", evr.Introduced.Revision); + Assert.NotNull(evr.Fixed); + Assert.Equal(1, evr.Fixed!.Epoch); + Assert.Equal("1.1.1n", evr.Fixed.UpstreamVersion); + Assert.Equal("0+deb11u5", evr.Fixed.Revision); + + var openPackage = Assert.Single(advisory.AffectedPackages, p => p.Platform == "bookworm"); + Assert.Empty(openPackage.VersionRanges); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html new file mode 100644 index 00000000..4048df6f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html @@ -0,0 +1,23 @@ + + + + DSA-2024-123 + + +

DSA-2024-123

+ + + + + +
NameDSA-2024-123
Descriptionopenssl - security update
SourceDebian
ReferencesCVE-2024-1000, CVE-2024-1001
+

Vulnerable and fixed packages

+ + + + + + +
Source PackageReleaseVersionStatus
opensslbookworm1:1.1.1n-0+deb11u2vulnerable
bookworm (security)1:1.1.1n-0+deb11u5fixed
trixie3.0.8-2vulnerable
trixie (security)3.0.12-1fixed
+ + diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html new file mode 100644 index 00000000..86fcceae --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html @@ -0,0 +1,21 @@ + + + + DSA-2024-124 + + +

DSA-2024-124

+ + + + + +
NameDSA-2024-124
Descriptionzlib - security update
SourceDebian
ReferencesCVE-2024-2000
+

Vulnerable and fixed packages

+ + + + +
Source PackageReleaseVersionStatus
zlibbookworm1:1.3.1-1vulnerable
trixie1:1.3.1-2vulnerable
+ + diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt new file mode 100644 index 00000000..6e9cf6c8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt @@ -0,0 +1,7 @@ +[12 Sep 2024] DSA-2024-123 openssl - security update + {CVE-2024-1000 CVE-2024-1001} + [bookworm] - openssl 1:1.1.1n-0+deb11u5 + [trixie] - openssl 3.0.12-1 +[10 Sep 2024] DSA-2024-124 zlib - security update + {CVE-2024-2000} + [bookworm] - zlib 1:1.3.2-1 diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj new file mode 100644 index 00000000..2ad7e300 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs new file mode 100644 index 00000000..53512498 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.Debian.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Class1.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Class1.cs deleted file mode 100644 index acbcc9b4..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Class1.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Distro.Debian; - -public sealed class DistroDebianConnectorPlugin : IConnectorPlugin -{ - public string Name => "distro-debian"; - - public bool IsAvailable(IServiceProvider services) => true; - - public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); - - private sealed class StubConnector : IFeedConnector - { - public StubConnector(string sourceName) => SourceName = sourceName; - - public string SourceName { get; } - - public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - } -} - diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs new file mode 100644 index 00000000..4f816458 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs @@ -0,0 +1,87 @@ +using System; + +namespace StellaOps.Feedser.Source.Distro.Debian.Configuration; + +public sealed class DebianOptions +{ + public const string HttpClientName = "feedser.debian"; + + /// + /// Raw advisory list published by the Debian security tracker team. + /// Defaults to the Salsa Git raw endpoint to avoid HTML scraping. + /// + public Uri ListEndpoint { get; set; } = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list"); + + /// + /// Base URI for advisory detail pages. Connector appends {AdvisoryId}. + /// + public Uri DetailBaseUri { get; set; } = new("https://security-tracker.debian.org/tracker/"); + + /// + /// Maximum advisories fetched per run to cap backfill effort. + /// + public int MaxAdvisoriesPerFetch { get; set; } = 40; + + /// + /// Initial history window pulled on first run. + /// + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + /// + /// Resume overlap to accommodate late edits of existing advisories. + /// + public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(2); + + /// + /// Request timeout used for list/detail fetches unless overridden via HTTP client. + /// + public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45); + + /// + /// Optional pacing delay between detail fetches. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero; + + /// + /// Custom user-agent for Debian tracker courtesy. + /// + public string UserAgent { get; set; } = "StellaOps.Feedser.Debian/0.1 (+https://stella-ops.org)"; + + public void Validate() + { + if (ListEndpoint is null || !ListEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("Debian list endpoint must be an absolute URI."); + } + + if (DetailBaseUri is null || !DetailBaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Debian detail base URI must be an absolute URI."); + } + + if (MaxAdvisoriesPerFetch <= 0 || MaxAdvisoriesPerFetch > 200) + { + throw new InvalidOperationException("MaxAdvisoriesPerFetch must be between 1 and 200."); + } + + if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365)) + { + throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days."); + } + + if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14)) + { + throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days."); + } + + if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes."); + } + + if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10)) + { + throw new InvalidOperationException("RequestDelay must be between 0 and 10 seconds."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs b/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs new file mode 100644 index 00000000..70e5d058 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs @@ -0,0 +1,637 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Distro.Debian.Configuration; +using StellaOps.Feedser.Source.Distro.Debian.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public sealed class DebianConnector : IFeedConnector +{ + private const string SchemaVersion = "debian.v1"; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly DebianOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + private static readonly Action LogMapped = + LoggerMessage.Define( + LogLevel.Information, + new EventId(1, "DebianMapped"), + "Debian advisory {AdvisoryId} mapped with {AffectedCount} packages"); + + public DebianConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => DebianConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var pendingDocuments = new HashSet(cursor.PendingDocuments); + var pendingMappings = new HashSet(cursor.PendingMappings); + var fetchCache = new Dictionary(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); + + var listUri = _options.ListEndpoint; + var listKey = listUri.ToString(); + touchedResources.Add(listKey); + + var existingList = await _documentStore.FindBySourceAndUriAsync(SourceName, listKey, cancellationToken).ConfigureAwait(false); + cursor.TryGetCache(listKey, out var cachedListEntry); + + var listRequest = new SourceFetchRequest(DebianOptions.HttpClientName, SourceName, listUri) + { + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["type"] = "index" + }, + AcceptHeaders = new[] { "text/plain", "text/plain; charset=utf-8" }, + TimeoutOverride = _options.FetchTimeout, + ETag = existingList?.Etag ?? cachedListEntry?.ETag, + LastModified = existingList?.LastModified ?? cachedListEntry?.LastModified, + }; + + SourceFetchResult listResult; + try + { + listResult = await _fetchService.FetchAsync(listRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Debian list fetch failed"); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var lastPublished = cursor.LastPublished ?? (now - _options.InitialBackfill); + var processedIds = new HashSet(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); + var newProcessedIds = new HashSet(StringComparer.OrdinalIgnoreCase); + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + var processedUpdated = false; + + if (listResult.IsNotModified) + { + if (existingList is not null) + { + fetchCache[listKey] = DebianFetchCacheEntry.FromDocument(existingList); + } + } + else if (listResult.IsSuccess && listResult.Document is not null) + { + fetchCache[listKey] = DebianFetchCacheEntry.FromDocument(listResult.Document); + + if (!listResult.Document.GridFsId.HasValue) + { + _logger.LogWarning("Debian list document {DocumentId} missing GridFS payload", listResult.Document.Id); + } + else + { + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(listResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download Debian list document {DocumentId}", listResult.Document.Id); + throw; + } + + var text = System.Text.Encoding.UTF8.GetString(bytes); + var entries = DebianListParser.Parse(text); + if (entries.Count > 0) + { + var windowStart = (cursor.LastPublished ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap; + if (windowStart < DateTimeOffset.UnixEpoch) + { + windowStart = DateTimeOffset.UnixEpoch; + } + + ProvenanceDiagnostics.ReportResumeWindow(SourceName, windowStart, _logger); + + var candidates = entries + .Where(entry => entry.Published >= windowStart) + .OrderBy(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (candidates.Count == 0) + { + candidates = entries + .OrderByDescending(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxAdvisoriesPerFetch) + .OrderBy(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + else if (candidates.Count > _options.MaxAdvisoriesPerFetch) + { + candidates = candidates + .OrderByDescending(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxAdvisoriesPerFetch) + .OrderBy(entry => entry.Published) + .ThenBy(entry => entry.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + foreach (var entry in candidates) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = new Uri(_options.DetailBaseUri, entry.AdvisoryId); + var cacheKey = detailUri.ToString(); + touchedResources.Add(cacheKey); + + cursor.TryGetCache(cacheKey, out var cachedDetail); + if (!fetchCache.TryGetValue(cacheKey, out var cachedInRun)) + { + cachedInRun = cachedDetail; + } + + var metadata = BuildDetailMetadata(entry); + var existingDetail = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest(DebianOptions.HttpClientName, SourceName, detailUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml" }, + TimeoutOverride = _options.FetchTimeout, + ETag = existingDetail?.Etag ?? cachedInRun?.ETag, + LastModified = existingDetail?.LastModified ?? cachedInRun?.LastModified, + }; + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to fetch Debian advisory {AdvisoryId}", entry.AdvisoryId); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.IsNotModified) + { + if (existingDetail is not null) + { + fetchCache[cacheKey] = DebianFetchCacheEntry.FromDocument(existingDetail); + if (string.Equals(existingDetail.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + pendingDocuments.Remove(existingDetail.Id); + pendingMappings.Remove(existingDetail.Id); + } + } + + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + fetchCache[cacheKey] = DebianFetchCacheEntry.FromDocument(result.Document); + pendingDocuments.Add(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + + if (entry.Published > maxPublished) + { + maxPublished = entry.Published; + newProcessedIds.Clear(); + processedUpdated = true; + } + + if (entry.Published == maxPublished) + { + newProcessedIds.Add(entry.AdvisoryId); + processedUpdated = true; + } + } + } + } + } + + if (fetchCache.Count > 0 && touchedResources.Count > 0) + { + var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in stale) + { + fetchCache.Remove(key); + } + } + + if (!processedUpdated && cursor.LastPublished.HasValue) + { + maxPublished = cursor.LastPublished.Value; + newProcessedIds = new HashSet(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithFetchCache(fetchCache); + + if (processedUpdated && maxPublished > DateTimeOffset.MinValue) + { + updatedCursor = updatedCursor.WithProcessed(maxPublished, newProcessedIds); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remaining = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remaining.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Debian document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + var metadata = ExtractMetadata(document); + if (metadata is null) + { + _logger.LogWarning("Debian document {DocumentId} missing required metadata", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download Debian document {DocumentId}", document.Id); + throw; + } + + var html = System.Text.Encoding.UTF8.GetString(bytes); + DebianAdvisoryDto dto; + try + { + dto = DebianHtmlParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse Debian advisory {AdvisoryId}", metadata.AdvisoryId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(document.Id); + continue; + } + + var payload = ToBson(dto); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, SchemaVersion, payload, _timeProvider.GetUtcNow()); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remaining.Remove(document.Id); + if (!pendingMappings.Contains(document.Id)) + { + pendingMappings.Add(document.Id); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remaining) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + DebianAdvisoryDto dto; + try + { + dto = FromBson(dtoRecord.Payload); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize Debian DTO for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = DebianMapper.Map(dto, document, _timeProvider.GetUtcNow()); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? DebianCursor.Empty : DebianCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(DebianCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static Dictionary BuildDetailMetadata(DebianListEntry entry) + { + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["debian.id"] = entry.AdvisoryId, + ["debian.published"] = entry.Published.ToString("O", CultureInfo.InvariantCulture), + ["debian.title"] = entry.Title, + ["debian.package"] = entry.SourcePackage + }; + + if (entry.CveIds.Count > 0) + { + metadata["debian.cves"] = string.Join(' ', entry.CveIds); + } + + return metadata; + } + + private static DebianDetailMetadata? ExtractMetadata(DocumentRecord document) + { + if (document.Metadata is null) + { + return null; + } + + if (!document.Metadata.TryGetValue("debian.id", out var id) || string.IsNullOrWhiteSpace(id)) + { + return null; + } + + if (!document.Metadata.TryGetValue("debian.published", out var publishedRaw) + || !DateTimeOffset.TryParse(publishedRaw, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published)) + { + published = document.FetchedAt; + } + + var title = document.Metadata.TryGetValue("debian.title", out var t) ? t : id; + var package = document.Metadata.TryGetValue("debian.package", out var pkg) && !string.IsNullOrWhiteSpace(pkg) + ? pkg + : id; + + IReadOnlyList cveList = Array.Empty(); + if (document.Metadata.TryGetValue("debian.cves", out var cvesRaw) && !string.IsNullOrWhiteSpace(cvesRaw)) + { + cveList = cvesRaw + .Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Select(static s => s!) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + return new DebianDetailMetadata( + id.Trim(), + new Uri(document.Uri, UriKind.Absolute), + published.ToUniversalTime(), + title, + package, + cveList); + } + + private static BsonDocument ToBson(DebianAdvisoryDto dto) + { + var packages = new BsonArray(); + foreach (var package in dto.Packages) + { + var packageDoc = new BsonDocument + { + ["package"] = package.Package, + ["release"] = package.Release, + ["status"] = package.Status, + }; + + if (!string.IsNullOrWhiteSpace(package.IntroducedVersion)) + { + packageDoc["introduced"] = package.IntroducedVersion; + } + + if (!string.IsNullOrWhiteSpace(package.FixedVersion)) + { + packageDoc["fixed"] = package.FixedVersion; + } + + if (!string.IsNullOrWhiteSpace(package.LastAffectedVersion)) + { + packageDoc["last"] = package.LastAffectedVersion; + } + + if (package.Published.HasValue) + { + packageDoc["published"] = package.Published.Value.UtcDateTime; + } + + packages.Add(packageDoc); + } + + var references = new BsonArray(dto.References.Select(reference => + { + var doc = new BsonDocument + { + ["url"] = reference.Url + }; + + if (!string.IsNullOrWhiteSpace(reference.Kind)) + { + doc["kind"] = reference.Kind; + } + + if (!string.IsNullOrWhiteSpace(reference.Title)) + { + doc["title"] = reference.Title; + } + + return doc; + })); + + return new BsonDocument + { + ["advisoryId"] = dto.AdvisoryId, + ["sourcePackage"] = dto.SourcePackage, + ["title"] = dto.Title, + ["description"] = dto.Description ?? string.Empty, + ["cves"] = new BsonArray(dto.CveIds), + ["packages"] = packages, + ["references"] = references, + }; + } + + private static DebianAdvisoryDto FromBson(BsonDocument document) + { + var advisoryId = document.GetValue("advisoryId", "").AsString; + var sourcePackage = document.GetValue("sourcePackage", advisoryId).AsString; + var title = document.GetValue("title", advisoryId).AsString; + var description = document.TryGetValue("description", out var desc) ? desc.AsString : null; + + var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray cvesBson + ? cvesBson.OfType() + .Select(static value => value.ToString()) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Select(static s => s!) + .ToArray() + : Array.Empty(); + + var packages = new List(); + if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray packagesBson) + { + foreach (var element in packagesBson.OfType()) + { + packages.Add(new DebianPackageStateDto( + element.GetValue("package", sourcePackage).AsString, + element.GetValue("release", string.Empty).AsString, + element.GetValue("status", "unknown").AsString, + element.TryGetValue("introduced", out var introducedValue) ? introducedValue.AsString : null, + element.TryGetValue("fixed", out var fixedValue) ? fixedValue.AsString : null, + element.TryGetValue("last", out var lastValue) ? lastValue.AsString : null, + element.TryGetValue("published", out var publishedValue) + ? publishedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => (DateTimeOffset?)null, + } + : null)); + } + } + + var references = new List(); + if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray refBson) + { + foreach (var element in refBson.OfType()) + { + references.Add(new DebianReferenceDto( + element.GetValue("url", "").AsString, + element.TryGetValue("kind", out var kind) ? kind.AsString : null, + element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null)); + } + } + + return new DebianAdvisoryDto( + advisoryId, + sourcePackage, + title, + description, + cves, + packages, + references); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs new file mode 100644 index 00000000..51afb4c3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs @@ -0,0 +1,22 @@ +using System; +using System.Threading; +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public sealed class DebianConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "distro-debian"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs new file mode 100644 index 00000000..04bf70b1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Distro.Debian.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public sealed class DebianDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:debian"; + private const string FetchSchedule = "*/30 * * * *"; + private const string ParseSchedule = "7,37 * * * *"; + private const string MapSchedule = "12,42 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddDebianConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob( + DebianJobKinds.Fetch, + cronExpression: FetchSchedule, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob( + DebianJobKinds.Parse, + cronExpression: ParseSchedule, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob( + DebianJobKinds.Map, + cronExpression: MapSchedule, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs new file mode 100644 index 00000000..185d8b1b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Distro.Debian.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public static class DebianServiceCollectionExtensions +{ + public static IServiceCollection AddDebianConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(DebianOptions.HttpClientName, (sp, httpOptions) => + { + var options = sp.GetRequiredService>().Value; + httpOptions.BaseAddress = options.DetailBaseUri.GetLeftPart(UriPartial.Authority) is { Length: > 0 } authority + ? new Uri(authority, UriKind.Absolute) + : new Uri("https://security-tracker.debian.org/", UriKind.Absolute); + httpOptions.Timeout = options.FetchTimeout; + httpOptions.UserAgent = options.UserAgent; + httpOptions.AllowedHosts.Clear(); + httpOptions.AllowedHosts.Add(options.DetailBaseUri.Host); + httpOptions.AllowedHosts.Add(options.ListEndpoint.Host); + httpOptions.DefaultRequestHeaders["Accept"] = "text/html,application/xhtml+xml,text/plain;q=0.9,application/json;q=0.8"; + }); + + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs new file mode 100644 index 00000000..033b9d44 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianAdvisoryDto( + string AdvisoryId, + string SourcePackage, + string? Title, + string? Description, + IReadOnlyList CveIds, + IReadOnlyList Packages, + IReadOnlyList References); + +internal sealed record DebianPackageStateDto( + string Package, + string Release, + string Status, + string? IntroducedVersion, + string? FixedVersion, + string? LastAffectedVersion, + DateTimeOffset? Published); + +internal sealed record DebianReferenceDto( + string Url, + string? Kind, + string? Title); diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs new file mode 100644 index 00000000..64f9c0d3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection ProcessedAdvisoryIds, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + private static readonly IReadOnlyCollection EmptyIds = Array.Empty(); + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static DebianCursor Empty { get; } = new(null, EmptyIds, EmptyGuidList, EmptyGuidList, EmptyCache); + + public static DebianCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastPublished = null; + if (document.TryGetValue("lastPublished", out var lastValue)) + { + lastPublished = lastValue.BsonType switch + { + BsonType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(), + BsonType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc), + _ => null, + }; + } + + var processed = ReadStringArray(document, "processedIds"); + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var cache = ReadCache(document); + + return new DebianCursor(lastPublished, processed, pendingDocuments, pendingMappings, cache); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + if (ProcessedAdvisoryIds.Count > 0) + { + document["processedIds"] = new BsonArray(ProcessedAdvisoryIds); + } + + if (FetchCache.Count > 0) + { + var cacheDoc = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDoc[key] = entry.ToBsonDocument(); + } + + document["fetchCache"] = cacheDoc; + } + + return document; + } + + public DebianCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public DebianCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public DebianCursor WithProcessed(DateTimeOffset published, IEnumerable ids) + => this with + { + LastPublished = published.ToUniversalTime(), + ProcessedAdvisoryIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyIds + }; + + public DebianCursor WithFetchCache(IDictionary? cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyCache }; + } + + return this with { FetchCache = new Dictionary(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public bool TryGetCache(string key, out DebianFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = DebianFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + + private static IReadOnlyCollection ReadStringArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyIds; + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String) + { + var str = element.AsString.Trim(); + if (!string.IsNullOrEmpty(str)) + { + list.Add(str); + } + } + } + + return list; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary ReadCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) + { + return EmptyCache; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entry) + { + cache[element.Name] = DebianFetchCacheEntry.FromBson(entry); + } + } + + return cache; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs new file mode 100644 index 00000000..cddfcddd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianDetailMetadata( + string AdvisoryId, + Uri DetailUri, + DateTimeOffset Published, + string Title, + string SourcePackage, + IReadOnlyList CveIds); diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs new file mode 100644 index 00000000..e5e5f220 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs @@ -0,0 +1,76 @@ +using System; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) +{ + public static DebianFetchCacheEntry Empty { get; } = new(null, null); + + public static DebianFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + => new(document.Etag, document.LastModified); + + public static DebianFetchCacheEntry FromBson(BsonDocument document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + string? etag = null; + DateTimeOffset? lastModified = null; + + if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String) + { + etag = etagValue.AsString; + } + + if (document.TryGetValue("lastModified", out var modifiedValue)) + { + lastModified = modifiedValue.BsonType switch + { + BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc), + _ => null, + }; + } + + return new DebianFetchCacheEntry(etag, lastModified); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (!string.IsNullOrWhiteSpace(ETag)) + { + document["etag"] = ETag; + } + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + return document; + } + + public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + { + if (document is null) + { + return false; + } + + if (!string.Equals(document.Etag, ETag, StringComparison.Ordinal)) + { + return false; + } + + if (LastModified.HasValue && document.LastModified.HasValue) + { + return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime; + } + + return !LastModified.HasValue && !document.LastModified.HasValue; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs new file mode 100644 index 00000000..06d0b388 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs @@ -0,0 +1,326 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal static class DebianHtmlParser +{ + public static DebianAdvisoryDto Parse(string html, DebianDetailMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + var parser = new HtmlParser(); + var document = parser.ParseDocument(html); + + var description = ExtractDescription(document) ?? metadata.Title; + var references = ExtractReferences(document, metadata); + var packages = ExtractPackages(document, metadata.SourcePackage, metadata.Published); + + return new DebianAdvisoryDto( + metadata.AdvisoryId, + metadata.SourcePackage, + metadata.Title, + description, + metadata.CveIds, + packages, + references); + } + + private static string? ExtractDescription(IHtmlDocument document) + { + foreach (var table in document.QuerySelectorAll("table")) + { + if (table is not IHtmlTableElement tableElement) + { + continue; + } + + foreach (var row in tableElement.Rows) + { + if (row.Cells.Length < 2) + { + continue; + } + + var header = row.Cells[0].TextContent?.Trim(); + if (string.Equals(header, "Description", StringComparison.OrdinalIgnoreCase)) + { + return NormalizeWhitespace(row.Cells[1].TextContent); + } + } + + // Only the first table contains the metadata rows we need. + break; + } + + return null; + } + + private static IReadOnlyList ExtractReferences(IHtmlDocument document, DebianDetailMetadata metadata) + { + var references = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + // Add canonical Debian advisory page. + var canonical = new Uri($"https://www.debian.org/security/{metadata.AdvisoryId.ToLowerInvariant()}"); + references.Add(new DebianReferenceDto(canonical.ToString(), "advisory", metadata.Title)); + seen.Add(canonical.ToString()); + + foreach (var link in document.QuerySelectorAll("a")) + { + var href = link.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + string resolved; + if (Uri.TryCreate(href, UriKind.Absolute, out var absolute)) + { + resolved = absolute.ToString(); + } + else if (Uri.TryCreate(metadata.DetailUri, href, out var relative)) + { + resolved = relative.ToString(); + } + else + { + continue; + } + + if (!seen.Add(resolved)) + { + continue; + } + + var text = NormalizeWhitespace(link.TextContent); + string? kind = null; + if (text.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) + { + kind = "cve"; + } + else if (resolved.Contains("debian.org/security", StringComparison.OrdinalIgnoreCase)) + { + kind = "advisory"; + } + + references.Add(new DebianReferenceDto(resolved, kind, text)); + } + + return references; + } + + private static IReadOnlyList ExtractPackages(IHtmlDocument document, string defaultPackage, DateTimeOffset published) + { + var table = FindPackagesTable(document); + if (table is null) + { + return Array.Empty(); + } + + var accumulators = new Dictionary(StringComparer.OrdinalIgnoreCase); + string currentPackage = defaultPackage; + + foreach (var body in table.Bodies) + { + foreach (var row in body.Rows) + { + if (row.Cells.Length < 4) + { + continue; + } + + var packageCell = NormalizeWhitespace(row.Cells[0].TextContent); + if (!string.IsNullOrWhiteSpace(packageCell)) + { + currentPackage = ExtractPackageName(packageCell); + } + + if (string.IsNullOrWhiteSpace(currentPackage)) + { + continue; + } + + var releaseRaw = NormalizeWhitespace(row.Cells[1].TextContent); + var versionRaw = NormalizeWhitespace(row.Cells[2].TextContent); + var statusRaw = NormalizeWhitespace(row.Cells[3].TextContent); + if (string.IsNullOrWhiteSpace(releaseRaw)) + { + continue; + } + + var release = NormalizeRelease(releaseRaw); + var key = $"{currentPackage}|{release}"; + if (!accumulators.TryGetValue(key, out var accumulator)) + { + accumulator = new PackageAccumulator(currentPackage, release, published); + accumulators[key] = accumulator; + } + + accumulator.Apply(statusRaw, versionRaw); + } + } + + return accumulators.Values + .Where(static acc => acc.ShouldEmit) + .Select(static acc => acc.ToDto()) + .OrderBy(static dto => dto.Release, StringComparer.OrdinalIgnoreCase) + .ThenBy(static dto => dto.Package, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IHtmlTableElement? FindPackagesTable(IHtmlDocument document) + { + foreach (var table in document.QuerySelectorAll("table")) + { + if (table is not IHtmlTableElement tableElement) + { + continue; + } + + var header = tableElement.Rows.FirstOrDefault(); + if (header is null || header.Cells.Length < 4) + { + continue; + } + + var firstHeader = NormalizeWhitespace(header.Cells[0].TextContent); + var secondHeader = NormalizeWhitespace(header.Cells[1].TextContent); + var thirdHeader = NormalizeWhitespace(header.Cells[2].TextContent); + if (string.Equals(firstHeader, "Source Package", StringComparison.OrdinalIgnoreCase) + && string.Equals(secondHeader, "Release", StringComparison.OrdinalIgnoreCase) + && string.Equals(thirdHeader, "Version", StringComparison.OrdinalIgnoreCase)) + { + return tableElement; + } + } + + return null; + } + + private static string NormalizeRelease(string release) + { + var trimmed = release.Trim(); + var parenthesisIndex = trimmed.IndexOf('('); + if (parenthesisIndex > 0) + { + trimmed = trimmed[..parenthesisIndex].Trim(); + } + + return trimmed; + } + + private static string ExtractPackageName(string value) + { + var trimmed = value.Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + if (string.IsNullOrWhiteSpace(trimmed)) + { + return value.Trim(); + } + + if (trimmed.EndsWith(")", StringComparison.Ordinal) && trimmed.Contains('(')) + { + trimmed = trimmed[..trimmed.IndexOf('(')]; + } + + return trimmed.Trim(); + } + + private static string NormalizeWhitespace(string value) + => string.IsNullOrWhiteSpace(value) + ? string.Empty + : string.Join(' ', value.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + + private sealed class PackageAccumulator + { + private readonly DateTimeOffset _published; + + public PackageAccumulator(string package, string release, DateTimeOffset published) + { + Package = package; + Release = release; + _published = published; + Status = "unknown"; + } + + public string Package { get; } + + public string Release { get; } + + public string Status { get; private set; } + + public string? IntroducedVersion { get; private set; } + + public string? FixedVersion { get; private set; } + + public string? LastAffectedVersion { get; private set; } + + public bool ShouldEmit => + !string.Equals(Status, "not_affected", StringComparison.OrdinalIgnoreCase) + || IntroducedVersion is not null + || FixedVersion is not null; + + public void Apply(string statusRaw, string versionRaw) + { + var status = statusRaw.ToLowerInvariant(); + var version = string.IsNullOrWhiteSpace(versionRaw) ? null : versionRaw.Trim(); + + if (status.Contains("fixed", StringComparison.OrdinalIgnoreCase)) + { + FixedVersion = version; + if (!string.Equals(Status, "open", StringComparison.OrdinalIgnoreCase)) + { + Status = "resolved"; + } + + return; + } + + if (status.Contains("vulnerable", StringComparison.OrdinalIgnoreCase) + || status.Contains("open", StringComparison.OrdinalIgnoreCase)) + { + IntroducedVersion ??= version; + if (!string.Equals(Status, "resolved", StringComparison.OrdinalIgnoreCase)) + { + Status = "open"; + } + + LastAffectedVersion = null; + return; + } + + if (status.Contains("not affected", StringComparison.OrdinalIgnoreCase) + || status.Contains("not vulnerable", StringComparison.OrdinalIgnoreCase)) + { + Status = "not_affected"; + IntroducedVersion = null; + FixedVersion = null; + LastAffectedVersion = null; + return; + } + + if (status.Contains("end-of-life", StringComparison.OrdinalIgnoreCase) || status.Contains("end of life", StringComparison.OrdinalIgnoreCase)) + { + Status = "end_of_life"; + return; + } + + Status = statusRaw; + } + + public DebianPackageStateDto ToDto() + => new( + Package: Package, + Release: Release, + Status: Status, + IntroducedVersion: IntroducedVersion, + FixedVersion: FixedVersion, + LastAffectedVersion: LastAffectedVersion, + Published: _published); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs new file mode 100644 index 00000000..5cd4b3de --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal sealed record DebianListEntry( + string AdvisoryId, + DateTimeOffset Published, + string Title, + string SourcePackage, + IReadOnlyList CveIds); diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs new file mode 100644 index 00000000..c56a85f9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal static class DebianListParser +{ + private static readonly Regex HeaderRegex = new("^\\[(?[^\\]]+)\\]\\s+(?DSA-\\d{4,}-\\d+)\\s+(?.+)$", RegexOptions.Compiled); + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static IReadOnlyList<DebianListEntry> Parse(string? content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return Array.Empty<DebianListEntry>(); + } + + var entries = new List<DebianListEntry>(); + var currentCves = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + DateTimeOffset currentDate = default; + string? currentId = null; + string? currentTitle = null; + string? currentPackage = null; + + foreach (var rawLine in content.Split('\n')) + { + var line = rawLine.TrimEnd('\r'); + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + if (line[0] == '[') + { + if (currentId is not null && currentTitle is not null && currentPackage is not null) + { + entries.Add(new DebianListEntry( + currentId, + currentDate, + currentTitle, + currentPackage, + currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves))); + } + + currentCves.Clear(); + currentId = null; + currentTitle = null; + currentPackage = null; + + var match = HeaderRegex.Match(line); + if (!match.Success) + { + continue; + } + + if (!DateTimeOffset.TryParseExact( + match.Groups["date"].Value, + new[] { "dd MMM yyyy", "d MMM yyyy" }, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out currentDate)) + { + continue; + } + + currentId = match.Groups["id"].Value.Trim(); + currentTitle = match.Groups["title"].Value.Trim(); + + var separatorIndex = currentTitle.IndexOf(" - ", StringComparison.Ordinal); + currentPackage = separatorIndex > 0 + ? currentTitle[..separatorIndex].Trim() + : currentTitle.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault(); + if (string.IsNullOrWhiteSpace(currentPackage)) + { + currentPackage = currentId; + } + + continue; + } + + if (line[0] == '{') + { + foreach (Match match in CveRegex.Matches(line)) + { + if (match.Success && !string.IsNullOrWhiteSpace(match.Value)) + { + currentCves.Add(match.Value.ToUpperInvariant()); + } + } + } + } + + if (currentId is not null && currentTitle is not null && currentPackage is not null) + { + entries.Add(new DebianListEntry( + currentId, + currentDate, + currentTitle, + currentPackage, + currentCves.Count == 0 ? Array.Empty<string>() : new List<string>(currentCves))); + } + + return entries; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs new file mode 100644 index 00000000..91e66638 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Distro.Debian.Internal; + +internal static class DebianMapper +{ + public static Advisory Map( + DebianAdvisoryDto dto, + DocumentRecord document, + DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var affectedPackages = BuildAffectedPackages(dto, recordedAt); + + var fetchProvenance = new AdvisoryProvenance( + DebianConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mappingProvenance = new AdvisoryProvenance( + DebianConnectorPlugin.SourceName, + "mapping", + dto.AdvisoryId, + recordedAt); + + return new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title ?? dto.AdvisoryId, + summary: dto.Description, + language: "en", + published: dto.Packages.Select(p => p.Published).Where(p => p.HasValue).Select(p => p!.Value).Cast<DateTimeOffset?>().DefaultIfEmpty(null).Min(), + modified: recordedAt, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty<CvssMetric>(), + provenance: new[] { fetchProvenance, mappingProvenance }); + } + + private static string[] BuildAliases(DebianAdvisoryDto dto) + { + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + if (!string.IsNullOrWhiteSpace(dto.AdvisoryId)) + { + aliases.Add(dto.AdvisoryId.Trim()); + } + + foreach (var cve in dto.CveIds ?? Array.Empty<string>()) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve.Trim()); + } + } + + return aliases.OrderBy(a => a, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static AdvisoryReference[] BuildReferences(DebianAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty<AdvisoryReference>(); + } + + var references = new List<AdvisoryReference>(); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + try + { + var provenance = new AdvisoryProvenance( + DebianConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt); + + references.Add(new AdvisoryReference( + reference.Url, + NormalizeReferenceKind(reference.Kind), + reference.Kind, + reference.Title, + provenance)); + } + catch (ArgumentException) + { + // Ignore malformed URLs while keeping the rest of the advisory intact. + } + } + + return references.Count == 0 + ? Array.Empty<AdvisoryReference>() + : references + .OrderBy(r => r.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeReferenceKind(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim().ToLowerInvariant() switch + { + "advisory" or "dsa" => "advisory", + "cve" => "cve", + "patch" => "patch", + _ => null, + }; + } + + private static AdvisoryProvenance BuildPackageProvenance(DebianPackageStateDto package, DateTimeOffset recordedAt) + => new(DebianConnectorPlugin.SourceName, "affected", $"{package.Package}:{package.Release}", recordedAt); + + private static IReadOnlyList<AffectedPackage> BuildAffectedPackages(DebianAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.Packages is null || dto.Packages.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + var packages = new List<AffectedPackage>(dto.Packages.Count); + foreach (var package in dto.Packages) + { + if (string.IsNullOrWhiteSpace(package.Package)) + { + continue; + } + + var provenance = new[] { BuildPackageProvenance(package, recordedAt) }; + var ranges = BuildVersionRanges(package, recordedAt); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Deb, + identifier: package.Package.Trim(), + platform: package.Release, + versionRanges: ranges, + statuses: Array.Empty<AffectedPackageStatus>(), + provenance: provenance)); + } + + return packages; + } + + private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(DebianPackageStateDto package, DateTimeOffset recordedAt) + { + var provenance = new AdvisoryProvenance( + DebianConnectorPlugin.SourceName, + "range", + $"{package.Package}:{package.Release}", + recordedAt); + + var introduced = package.IntroducedVersion; + var fixedVersion = package.FixedVersion; + var lastAffected = package.LastAffectedVersion; + + if (string.IsNullOrWhiteSpace(introduced) && string.IsNullOrWhiteSpace(fixedVersion) && string.IsNullOrWhiteSpace(lastAffected)) + { + return Array.Empty<AffectedVersionRange>(); + } + + var extensions = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["debian.release"] = package.Release, + ["debian.status"] = package.Status + }; + + AddExtension(extensions, "debian.introduced", introduced); + AddExtension(extensions, "debian.fixed", fixedVersion); + AddExtension(extensions, "debian.lastAffected", lastAffected); + + var primitives = BuildEvrPrimitives(introduced, fixedVersion, lastAffected); + return new[] + { + new AffectedVersionRange( + rangeKind: "evr", + introducedVersion: introduced, + fixedVersion: fixedVersion, + lastAffectedVersion: lastAffected, + rangeExpression: BuildRangeExpression(introduced, fixedVersion, lastAffected), + provenance: provenance, + primitives: primitives is null && extensions.Count == 0 + ? null + : new RangePrimitives( + SemVer: null, + Nevra: null, + Evr: primitives, + VendorExtensions: extensions.Count == 0 ? null : extensions)) + }; + } + + private static EvrPrimitive? BuildEvrPrimitives(string? introduced, string? fixedVersion, string? lastAffected) + { + var introducedComponent = ParseEvr(introduced); + var fixedComponent = ParseEvr(fixedVersion); + var lastAffectedComponent = ParseEvr(lastAffected); + + if (introducedComponent is null && fixedComponent is null && lastAffectedComponent is null) + { + return null; + } + + return new EvrPrimitive(introducedComponent, fixedComponent, lastAffectedComponent); + } + + private static EvrComponent? ParseEvr(string? value) + { + if (!DebianEvr.TryParse(value, out var evr) || evr is null) + { + return null; + } + + return new EvrComponent( + evr.Epoch, + evr.Version, + evr.Revision.Length == 0 ? null : evr.Revision); + } + + private static string? BuildRangeExpression(string? introduced, string? fixedVersion, string? lastAffected) + { + var parts = new List<string>(); + if (!string.IsNullOrWhiteSpace(introduced)) + { + parts.Add($"introduced:{introduced.Trim()}"); + } + + if (!string.IsNullOrWhiteSpace(fixedVersion)) + { + parts.Add($"fixed:{fixedVersion.Trim()}"); + } + + if (!string.IsNullOrWhiteSpace(lastAffected)) + { + parts.Add($"last:{lastAffected.Trim()}"); + } + + return parts.Count == 0 ? null : string.Join(" ", parts); + } + + private static void AddExtension(IDictionary<string, string> extensions, string key, string? value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + extensions[key] = value.Trim(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs new file mode 100644 index 00000000..871168da --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +internal static class DebianJobKinds +{ + public const string Fetch = "source:debian:fetch"; + public const string Parse = "source:debian:parse"; + public const string Map = "source:debian:map"; +} + +internal sealed class DebianFetchJob : IJob +{ + private readonly DebianConnector _connector; + + public DebianFetchJob(DebianConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class DebianParseJob : IJob +{ + private readonly DebianConnector _connector; + + public DebianParseJob(DebianConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class DebianMapJob : IJob +{ + private readonly DebianConnector _connector; + + public DebianMapJob(DebianConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj b/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj index 182529d4..34c6b8e9 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj +++ b/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj @@ -11,6 +11,7 @@ <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> </ItemGroup> </Project> - diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json index 2e245d76..7ed28f72 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json @@ -44,6 +44,28 @@ "fixedVersion": "kernel-0:4.18.0-513.5.1.el8.x86_64", "introducedVersion": null, "lastAffectedVersion": "kernel-0:4.18.0-500.1.0.el8.x86_64", + "primitives": { + "evr": null, + "nevra": { + "fixed": { + "architecture": "x86_64", + "epoch": 0, + "name": "kernel", + "release": "513.5.1.el8", + "version": "4.18.0" + }, + "introduced": null, + "lastAffected": { + "architecture": "x86_64", + "epoch": 0, + "name": "kernel", + "release": "500.1.0.el8", + "version": "4.18.0" + } + }, + "semVer": null, + "vendorExtensions": null + }, "provenance": { "kind": "package.nevra", "recordedAt": "2025-10-05T00:00:00+00:00", diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json new file mode 100644 index 00000000..d32c2300 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json @@ -0,0 +1,110 @@ +{ + "advisoryKey": "RHSA-2025:0002", + "affectedPackages": [ + { + "identifier": "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 9", + "provenance": [ + { + "kind": "oval", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + } + ], + "statuses": [ + { + "provenance": { + "kind": "oval", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + }, + "status": "known_not_affected" + }, + { + "provenance": { + "kind": "oval", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + }, + "status": "under_investigation" + } + ], + "type": "cpe", + "versionRanges": [] + }, + { + "identifier": "kernel-0:5.14.0-400.el9.x86_64", + "platform": "Red Hat Enterprise Linux 9", + "provenance": [ + { + "kind": "package.nevra", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "kernel-0:5.14.0-400.el9.x86_64" + } + ], + "statuses": [ + { + "provenance": { + "kind": "package.nevra", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "kernel-0:5.14.0-400.el9.x86_64" + }, + "status": "known_not_affected" + } + ], + "type": "rpm", + "versionRanges": [] + } + ], + "aliases": [ + "CVE-2025-0002", + "RHSA-2025:0002" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-05T12:00:00+00:00", + "provenance": [ + { + "kind": "advisory", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "RHSA-2025:0002" + } + ], + "published": "2025-10-05T12:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "kind": "reference", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "https://access.redhat.com/errata/RHSA-2025:0002" + }, + "sourceTag": null, + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0002" + }, + { + "kind": "external", + "provenance": { + "kind": "reference", + "recordedAt": "2025-10-05T12:00:00+00:00", + "source": "redhat", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0002" + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0002" + } + ], + "severity": "medium", + "summary": "Second advisory covering unaffected packages.", + "title": "Red Hat Security Advisory: Follow-up kernel status" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json new file mode 100644 index 00000000..6887f433 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json @@ -0,0 +1,113 @@ +{ + "advisoryKey": "RHSA-2025:0003", + "affectedPackages": [ + { + "identifier": "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 9", + "provenance": [ + { + "kind": "oval", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + } + ], + "statuses": [ + { + "provenance": { + "kind": "oval", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "9Base-RHEL-9" + }, + "status": "known_affected" + } + ], + "type": "cpe", + "versionRanges": [] + } + ], + "aliases": [ + "CVE-2025-0003", + "RHSA-2025:0003" + ], + "cvssMetrics": [ + { + "baseScore": 7.5, + "baseSeverity": "high", + "provenance": { + "kind": "cvss", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "CVE-2025-0003" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-06T09:00:00+00:00", + "provenance": [ + { + "kind": "advisory", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "RHSA-2025:0003" + } + ], + "published": "2025-10-06T09:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "kind": "reference", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + "sourceTag": null, + "summary": "Primary advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + { + "kind": "mitigation", + "provenance": { + "kind": "reference", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "https://access.redhat.com/solutions/999999" + }, + "sourceTag": null, + "summary": "Knowledge base guidance", + "url": "https://access.redhat.com/solutions/999999" + }, + { + "kind": "exploit", + "provenance": { + "kind": "reference", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" + }, + "sourceTag": null, + "summary": "Exploit tracking", + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" + }, + { + "kind": "external", + "provenance": { + "kind": "reference", + "recordedAt": "2025-10-06T09:00:00+00:00", + "source": "redhat", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + } + ], + "severity": "high", + "summary": "Advisory with mixed reference sources to verify dedupe ordering.", + "title": "Red Hat Security Advisory: Reference dedupe validation" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs index bee9dc00..3f476233 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs @@ -1,10 +1,12 @@ using System; using System.Globalization; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; +using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; @@ -19,10 +21,12 @@ using StellaOps.Feedser.Source.Common.Http; using StellaOps.Feedser.Source.Common.Testing; using StellaOps.Feedser.Source.Distro.RedHat; using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Feedser.Source.Distro.RedHat.Internal; using StellaOps.Feedser.Models; using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.Storage.Mongo.Advisories; using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; using StellaOps.Feedser.Testing; using StellaOps.Plugin; using Xunit; @@ -39,6 +43,7 @@ public sealed class RedHatConnectorTests : IAsyncLifetime private readonly CannedHttpMessageHandler _handler; private readonly ITestOutputHelper _output; private ServiceProvider? _serviceProvider; + private const bool ForceUpdateGoldens = false; public RedHatConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) { @@ -140,6 +145,10 @@ public sealed class RedHatConnectorTests : IAsyncLifetime rpmPackage.VersionRanges, range => string.Equals(range.FixedVersion, "kernel-0:4.18.0-513.5.1.el8.x86_64", StringComparison.Ordinal)); Assert.Equal("kernel-0:4.18.0-500.1.0.el8.x86_64", fixedRange.LastAffectedVersion); + var nevraPrimitive = fixedRange.Primitives?.Nevra; + Assert.NotNull(nevraPrimitive); + Assert.Null(nevraPrimitive!.Introduced); + Assert.Equal("kernel", nevraPrimitive.Fixed?.Name); var cpePackage = advisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Cpe); Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", cpePackage.Identifier); @@ -147,16 +156,22 @@ public sealed class RedHatConnectorTests : IAsyncLifetime Assert.Contains(advisory.References, reference => reference.Url == "https://access.redhat.com/errata/RHSA-2025:0001"); Assert.Contains(advisory.References, reference => reference.Url == "https://www.cve.org/CVERecord?id=CVE-2025-0001"); - var snapshot = SnapshotSerializer.ToSnapshot(advisory); + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n"); _output.WriteLine("-- RHSA-2025:0001 snapshot --\n" + snapshot); - var snapshotPath = Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "RedHat", "Fixtures", "rhsa-2025-0001.snapshot.json"); + var snapshotPath = ProjectFixturePath("rhsa-2025-0001.snapshot.json"); + if (ShouldUpdateGoldens()) + { + File.WriteAllText(snapshotPath, snapshot); + return; + } + var expectedSnapshot = File.ReadAllText(snapshotPath); Assert.Equal(NormalizeLineEndings(expectedSnapshot), NormalizeLineEndings(snapshot)); var state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); - Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); - Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings) && pendingMappings.AsBsonArray.Count == 0); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs2) && pendingDocs2.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings2) && pendingMappings2.AsBsonArray.Count == 0); const string fetchKind = "source:redhat:fetch"; const string parseKind = "source:redhat:parse"; @@ -226,8 +241,43 @@ public sealed class RedHatConnectorTests : IAsyncLifetime state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); - Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out pendingDocs) && pendingDocs.AsBsonArray.Count == 0); - Assert.True(state.Cursor.TryGetValue("pendingMappings", out pendingMappings) && pendingMappings.AsBsonArray.Count == 0); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs3) && pendingDocs3.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings3) && pendingMappings3.AsBsonArray.Count == 0); + } + + [Fact] + public void GoldenFixturesMatchSnapshots() + { + var fixtures = new[] + { + new GoldenFixtureCase( + AdvisoryId: "RHSA-2025:0002", + InputFile: "csaf-rhsa-2025-0002.json", + SnapshotFile: "rhsa-2025-0002.snapshot.json", + ValidatedAt: DateTimeOffset.Parse("2025-10-05T12:00:00Z")), + new GoldenFixtureCase( + AdvisoryId: "RHSA-2025:0003", + InputFile: "csaf-rhsa-2025-0003.json", + SnapshotFile: "rhsa-2025-0003.snapshot.json", + ValidatedAt: DateTimeOffset.Parse("2025-10-06T09:00:00Z")), + }; + + var updateGoldens = ShouldUpdateGoldens(); + + foreach (var fixture in fixtures) + { + var snapshot = MapFixtureToSnapshot(fixture); + var snapshotPath = ProjectFixturePath(fixture.SnapshotFile); + + if (updateGoldens) + { + File.WriteAllText(snapshotPath, snapshot); + continue; + } + + var expected = File.ReadAllText(snapshotPath).Replace("\r\n", "\n"); + Assert.Equal(expected, snapshot); + } } [Fact] @@ -400,8 +450,78 @@ public sealed class RedHatConnectorTests : IAsyncLifetime Assert.Equal("https://www.cve.org/CVERecord?id=CVE-2025-0003", reference.Url); Assert.Equal("CVE record", reference.Summary); }); + Assert.Equal(4, references.Length); + + Assert.Equal("self", references[0].Kind); + Assert.Equal("https://access.redhat.com/errata/RHSA-2025:0003", references[0].Url); + Assert.Equal("Primary advisory", references[0].Summary); + + Assert.Equal("mitigation", references[1].Kind); + Assert.Equal("https://access.redhat.com/solutions/999999", references[1].Url); + Assert.Equal("Knowledge base guidance", references[1].Summary); + + Assert.Equal("exploit", references[2].Kind); + Assert.Equal("https://bugzilla.redhat.com/show_bug.cgi?id=2222222", references[2].Url); + + Assert.Equal("external", references[3].Kind); + Assert.Equal("https://www.cve.org/CVERecord?id=CVE-2025-0003", references[3].Url); + Assert.Equal("CVE record", references[3].Summary); } + private static string MapFixtureToSnapshot(GoldenFixtureCase fixture) + { + var jsonPath = ProjectFixturePath(fixture.InputFile); + var json = File.ReadAllText(jsonPath); + + using var jsonDocument = JsonDocument.Parse(json); + var bson = BsonDocument.Parse(json); + + var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) + { + ["advisoryId"] = fixture.AdvisoryId, + }; + + var document = new DocumentRecord( + Guid.NewGuid(), + RedHatConnectorPlugin.SourceName, + $"https://access.redhat.com/hydra/rest/securitydata/csaf/{fixture.AdvisoryId}.json", + fixture.ValidatedAt, + new string('0', 64), + DocumentStatuses.Mapped, + "application/json", + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: fixture.ValidatedAt, + GridFsId: null); + + var dto = new DtoRecord(Guid.NewGuid(), document.Id, RedHatConnectorPlugin.SourceName, "redhat.csaf.v2", bson, fixture.ValidatedAt); + + var advisory = RedHatMapper.Map(RedHatConnectorPlugin.SourceName, dto, document, jsonDocument); + Assert.NotNull(advisory); + + return SnapshotSerializer.ToSnapshot(advisory!).Replace("\r\n", "\n"); + } + + private static bool ShouldUpdateGoldens() + => ForceUpdateGoldens + || IsTruthy(Environment.GetEnvironmentVariable("UPDATE_GOLDENS")) + || IsTruthy(Environment.GetEnvironmentVariable("DOTNET_TEST_UPDATE_GOLDENS")); + + private static bool IsTruthy(string? value) + => !string.IsNullOrWhiteSpace(value) + && (string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "yes", StringComparison.OrdinalIgnoreCase)); + + private sealed record GoldenFixtureCase(string AdvisoryId, string InputFile, string SnapshotFile, DateTimeOffset ValidatedAt); + + private static string ProjectFixturePath(string filename) + => Path.Combine(GetProjectRoot(), "RedHat", "Fixtures", filename); + + private static string GetProjectRoot() + => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..")); + private async Task EnsureServiceProviderAsync(RedHatOptions options) { if (_serviceProvider is not null) @@ -492,9 +612,25 @@ public sealed class RedHatConnectorTests : IAsyncLifetime } private static string ReadFixture(string name) + => File.ReadAllText(ResolveFixturePath(name)); + + private static string ResolveFixturePath(string filename) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "RedHat", "Fixtures", name); - return File.ReadAllText(path); + var candidates = new[] + { + Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "RedHat", "Fixtures", filename), + Path.Combine(AppContext.BaseDirectory, "RedHat", "Fixtures", filename), + }; + + foreach (var candidate in candidates) + { + if (File.Exists(candidate)) + { + return candidate; + } + } + + throw new FileNotFoundException($"Fixture '{filename}' not found in output directory.", filename); } private static string NormalizeLineEndings(string value) diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md b/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md index 50dfbcef..ac6ee1ba 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md @@ -19,10 +19,9 @@ Red Hat distro connector (Security Data API and OVAL) providing authoritative OS In: authoritative rpm ranges, RHSA mapping, OVAL interpretation, watermarking. Out: building RPM artifacts; cross-distro reconciliation beyond Red Hat. ## Observability & security expectations -- Metrics: redhat.fetch.items, redhat.oval.defs, redhat.parse.fail, redhat.map.affected_rpm. +- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged `feedser.source=redhat`, capturing fetch volumes, parse/OVAL failures, and map affected counts without bespoke metric names. - Logs: cursor bounds, advisory ids, NEVRA counts; allowlist Red Hat endpoints. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Distro.RedHat.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs index fdf211c0..abf8c827 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs @@ -6,6 +6,7 @@ using System.Text.Json.Serialization; using StellaOps.Feedser.Models; using StellaOps.Feedser.Source.Distro.RedHat.Internal.Models; using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Distro; using StellaOps.Feedser.Normalization.Identifiers; using StellaOps.Feedser.Normalization.Text; using StellaOps.Feedser.Storage.Mongo.Documents; @@ -310,14 +311,28 @@ internal static class RedHatMapper if (rpm.Statuses.Contains(RedHatProductStatuses.Fixed) || rpm.Statuses.Contains(RedHatProductStatuses.FirstFixed)) { - ranges.Add(new AffectedVersionRange("nevra", null, rpm.Nevra, lastKnownAffected, null, provenance)); + ranges.Add(new AffectedVersionRange( + "nevra", + introducedVersion: null, + fixedVersion: rpm.Nevra, + lastAffectedVersion: lastKnownAffected, + rangeExpression: null, + provenance: provenance, + primitives: BuildNevraPrimitives(null, rpm.Nevra, lastKnownAffected))); } if (!rpm.Statuses.Contains(RedHatProductStatuses.Fixed) && !rpm.Statuses.Contains(RedHatProductStatuses.FirstFixed) && rpm.Statuses.Contains(RedHatProductStatuses.KnownAffected)) { - ranges.Add(new AffectedVersionRange("nevra", null, null, rpm.Nevra, null, provenance)); + ranges.Add(new AffectedVersionRange( + "nevra", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: rpm.Nevra, + rangeExpression: null, + provenance: provenance, + primitives: BuildNevraPrimitives(null, null, rpm.Nevra))); } if (rpm.Statuses.Contains(RedHatProductStatuses.KnownNotAffected)) @@ -605,6 +620,31 @@ internal static class RedHatMapper : null; } + private static RangePrimitives BuildNevraPrimitives(string? introduced, string? fixedVersion, string? lastAffected) + { + var primitive = new NevraPrimitive( + ParseNevraComponent(introduced), + ParseNevraComponent(fixedVersion), + ParseNevraComponent(lastAffected)); + + return new RangePrimitives(null, primitive, null, null); + } + + private static NevraComponent? ParseNevraComponent(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (!Nevra.TryParse(value, out var parsed) || parsed is null) + { + return null; + } + + return new NevraComponent(parsed.Name, parsed.Epoch, parsed.Version, parsed.Release, parsed.Architecture); + } + private static string? NormalizeId(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..e49dac90 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.RedHat.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs index 57a4a3af..156f5050 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs @@ -72,6 +72,8 @@ public sealed class RedHatConnector : IFeedConnector afterThreshold = DateTimeOffset.UnixEpoch; } + ProvenanceDiagnostics.ReportResumeWindow(SourceName, afterThreshold, _logger); + var processedSet = new HashSet<string>(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); var newSummaries = new List<RedHatSummaryItem>(); var stopDueToOlderData = false; diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md b/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md index 7e02c217..564cfa2c 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md @@ -3,12 +3,12 @@ |---|---|---|---| |Hydra fetch with after= cursor|BE-Conn-RH|Source.Common|**DONE** – windowed paging with overlap, ETag/Last-Modified persisted.| |DTOs for Security Data + OVAL|BE-Conn-RH|Tests|**DONE** – CSAF payloads serialized into `redhat.csaf.v2` DTOs.| -|NEVRA parser/comparer (complete)|BE-Conn-RH|Models|Covered by NevraTests; add edge cases.| +|NEVRA parser/comparer (complete)|BE-Conn-RH|Models|**DONE** – parser/comparer shipped with coverage; add edge cases as needed.| |Mapper to canonical rpm/cpe affected|BE-Conn-RH|Models|**DONE** – maps fixed/known ranges, CPE provenance, status ranges.| |Job scheduler registration aligns with Options pipeline|BE-Conn-RH|Core|**DONE** – registered fetch/parse/map via JobSchedulerBuilder, preserving option overrides and tightening cron/timeouts.| |Watermark persistence + resume|BE-Conn-RH|Storage.Mongo|**DONE** – cursor updates via SourceStateRepository.| |Precedence tests vs NVD|QA|Merge|**DONE** – Added AffectedPackagePrecedenceResolver + tests ensuring Red Hat CPEs override NVD ranges.| -|Golden mapping fixtures|QA|Fixtures|**DONE** – fixtures refreshed; RedHat connector tests updated and passing under new deterministic outputs.| +|Golden mapping fixtures|QA|Fixtures|**DONE** – fixture validation test now snapshots RHSA-2025:0001/0002/0003 with env-driven regeneration.| |Job scheduling defaults for source:redhat tasks|BE-Core|JobScheduler|**DONE** – Cron windows + per-job timeouts defined for fetch/parse/map.| |Express unaffected/investigation statuses without overloading range fields|BE-Conn-RH|Models|**DONE** – Introduced AffectedPackageStatus collection and updated mapper/tests.| |Reference dedupe & ordering in mapper|BE-Conn-RH|Models|DONE – mapper consolidates by URL, merges metadata, deterministic ordering validated in tests.| diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv new file mode 100644 index 00000000..93c8eeca --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv @@ -0,0 +1,2 @@ +"suse-su-2025_0001-1.json","2025-01-21T10:00:00Z" +"suse-su-2025_0002-1.json","2025-01-22T08:30:00Z" diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json new file mode 100644 index 00000000..d3888f08 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json @@ -0,0 +1,63 @@ +{ + "document": { + "title": "openssl - security update", + "tracking": { + "id": "SUSE-SU-2025:0001-1", + "initial_release_date": "2025-01-21T00:00:00Z", + "current_release_date": "2025-01-21T00:00:00Z" + }, + "references": [ + { + "category": "self", + "summary": "SUSE notice", + "url": "https://www.suse.com/security/cve/CVE-2025-0001/" + } + ], + "notes": [ + { + "category": "summary", + "text": "Security update for openssl" + } + ] + }, + "product_tree": { + "branches": [ + { + "category": "vendor", + "name": "SUSE", + "branches": [ + { + "category": "product_family", + "name": "SUSE Linux Enterprise Server 15 SP5", + "branches": [ + { + "category": "architecture", + "name": "x86_64", + "branches": [ + { + "category": "product_version", + "name": "openssl-1.1.1w-150500.17.25.1.x86_64", + "product": { + "name": "openssl-1.1.1w-150500.17.25.1.x86_64", + "product_id": "SUSE Linux Enterprise Server 15 SP5:openssl-1.1.1w-150500.17.25.1.x86_64" + } + } + ] + } + ] + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0001", + "product_status": { + "recommended": [ + "SUSE Linux Enterprise Server 15 SP5:openssl-1.1.1w-150500.17.25.1.x86_64" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json new file mode 100644 index 00000000..b692233f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json @@ -0,0 +1,66 @@ +{ + "document": { + "title": "postgresql - investigation update", + "tracking": { + "id": "SUSE-SU-2025:0002-1", + "initial_release_date": "2025-01-22T00:00:00Z", + "current_release_date": "2025-01-22T00:00:00Z" + }, + "references": [ + { + "category": "external", + "summary": "Upstream CVE", + "url": "https://www.postgresql.org/support/security/CVE-2025-0002/" + } + ], + "notes": [ + { + "category": "summary", + "text": "Investigation ongoing for postgresql security issue." + } + ] + }, + "product_tree": { + "branches": [ + { + "category": "vendor", + "name": "SUSE", + "branches": [ + { + "category": "product_family", + "name": "openSUSE Tumbleweed", + "branches": [ + { + "category": "architecture", + "name": "x86_64", + "branches": [ + { + "category": "product_version", + "name": "postgresql16-16.3-2.1.x86_64", + "product": { + "name": "postgresql16-16.3-2.1.x86_64", + "product_id": "openSUSE Tumbleweed:postgresql16-16.3-2.1.x86_64" + } + } + ] + } + ] + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0002", + "product_status": { + "known_affected": [ + "openSUSE Tumbleweed:postgresql16-16.3-2.1.x86_64" + ], + "under_investigation": [ + "openSUSE Tumbleweed:postgresql16-16.3-2.1.x86_64" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj new file mode 100644 index 00000000..0e81cdc8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Update="Source\Distro\Suse\Fixtures\**\*"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs new file mode 100644 index 00000000..5e87bdd4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs @@ -0,0 +1,168 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.Suse; +using StellaOps.Feedser.Source.Distro.Suse.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Distro.Suse.Tests; + +[Collection("mongo-fixture")] +public sealed class SuseConnectorTests : IAsyncLifetime +{ + private static readonly Uri ChangesUri = new("https://ftp.suse.com/pub/projects/security/csaf/changes.csv"); + private static readonly Uri AdvisoryResolvedUri = new("https://ftp.suse.com/pub/projects/security/csaf/suse-su-2025_0001-1.json"); + private static readonly Uri AdvisoryOpenUri = new("https://ftp.suse.com/pub/projects/security/csaf/suse-su-2025_0002-1.json"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public SuseConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 22, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProcessesResolvedAndOpenNotices() + { + await using var provider = await BuildServiceProviderAsync(); + + SeedInitialResponses(); + + var connector = provider.GetRequiredService<SuseConnector>(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var resolved = advisories.Single(a => a.AdvisoryKey == "SUSE-SU-2025:0001-1"); + var resolvedPackage = Assert.Single(resolved.AffectedPackages); + var resolvedRange = Assert.Single(resolvedPackage.VersionRanges); + Assert.Equal("nevra", resolvedRange.RangeKind); + Assert.NotNull(resolvedRange.Primitives); + Assert.NotNull(resolvedRange.Primitives!.Nevra?.Fixed); + + var open = advisories.Single(a => a.AdvisoryKey == "SUSE-SU-2025:0002-1"); + var openPackage = Assert.Single(open.AffectedPackages); + Assert.Equal("open", openPackage.Statuses.Single().Status); + + SeedNotModifiedResponses(); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + _handler.AssertNoPendingResponses(); + } + + private async Task<ServiceProvider> BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton<TimeProvider>(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddSuseConnector(options => + { + options.ChangesEndpoint = ChangesUri; + options.AdvisoryBaseUri = new Uri("https://ftp.suse.com/pub/projects/security/csaf/"); + options.MaxAdvisoriesPerFetch = 5; + options.RequestDelay = TimeSpan.Zero; + }); + + services.Configure<HttpClientFactoryOptions>(SuseOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + _handler.AddResponse(ChangesUri, () => BuildResponse(HttpStatusCode.OK, "suse-changes.csv", "\"changes-v1\"")); + _handler.AddResponse(AdvisoryResolvedUri, () => BuildResponse(HttpStatusCode.OK, "suse-su-2025_0001-1.json", "\"adv-1\"")); + _handler.AddResponse(AdvisoryOpenUri, () => BuildResponse(HttpStatusCode.OK, "suse-su-2025_0002-1.json", "\"adv-2\"")); + } + + private void SeedNotModifiedResponses() + { + _handler.AddResponse(ChangesUri, () => BuildResponse(HttpStatusCode.NotModified, "suse-changes.csv", "\"changes-v1\"")); + _handler.AddResponse(AdvisoryResolvedUri, () => BuildResponse(HttpStatusCode.NotModified, "suse-su-2025_0001-1.json", "\"adv-1\"")); + _handler.AddResponse(AdvisoryOpenUri, () => BuildResponse(HttpStatusCode.NotModified, "suse-su-2025_0002-1.json", "\"adv-2\"")); + } + + private HttpResponseMessage BuildResponse(HttpStatusCode statusCode, string fixture, string etag) + { + var response = new HttpResponseMessage(statusCode); + if (statusCode == HttpStatusCode.OK) + { + var contentType = fixture.EndsWith(".csv", StringComparison.OrdinalIgnoreCase) ? "text/csv" : "application/json"; + response.Content = new StringContent(ReadFixture(Path.Combine("Source", "Distro", "Suse", "Fixtures", fixture)), Encoding.UTF8, contentType); + } + + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath.Replace('/', Path.DirectorySeparatorChar)); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Fixture '{relativePath}' not found.", path); + } + + return File.ReadAllText(path); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs new file mode 100644 index 00000000..f0ed49d5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using StellaOps.Feedser.Source.Distro.Suse.Internal; +using Xunit; + +namespace StellaOps.Feedser.Source.Distro.Suse.Tests; + +public sealed class SuseCsafParserTests +{ + [Fact] + public void Parse_ProducesRecommendedAndAffectedPackages() + { + var json = ReadFixture("Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json"); + var dto = SuseCsafParser.Parse(json); + + Assert.Equal("SUSE-SU-2025:0001-1", dto.AdvisoryId); + Assert.Contains("CVE-2025-0001", dto.CveIds); + var package = Assert.Single(dto.Packages); + Assert.Equal("openssl", package.Package); + Assert.Equal("resolved", package.Status); + Assert.NotNull(package.FixedVersion); + Assert.Equal("SUSE Linux Enterprise Server 15 SP5", package.Platform); + Assert.Equal("openssl-1.1.1w-150500.17.25.1.x86_64", package.CanonicalNevra); + } + + [Fact] + public void Parse_HandlesOpenInvestigation() + { + var json = ReadFixture("Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json"); + var dto = SuseCsafParser.Parse(json); + + Assert.Equal("SUSE-SU-2025:0002-1", dto.AdvisoryId); + Assert.Contains("CVE-2025-0002", dto.CveIds); + var package = Assert.Single(dto.Packages); + Assert.Equal("open", package.Status); + Assert.Equal("postgresql16", package.Package); + Assert.NotNull(package.LastAffectedVersion); + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath.Replace('/', Path.DirectorySeparatorChar)); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Fixture '{relativePath}' not found.", path); + } + + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs new file mode 100644 index 00000000..29e45b68 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.IO; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Distro.Suse; +using StellaOps.Feedser.Source.Distro.Suse.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using Xunit; + +namespace StellaOps.Feedser.Source.Distro.Suse.Tests; + +public sealed class SuseMapperTests +{ + [Fact] + public void Map_BuildsNevraRangePrimitives() + { + var json = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "Suse", "Fixtures", "suse-su-2025_0001-1.json")); + var dto = SuseCsafParser.Parse(json); + + var document = new DocumentRecord( + Guid.NewGuid(), + SuseConnectorPlugin.SourceName, + "https://ftp.suse.com/pub/projects/security/csaf/suse-su-2025_0001-1.json", + DateTimeOffset.UtcNow, + "sha256", + DocumentStatuses.PendingParse, + "application/json", + Headers: null, + Metadata: new Dictionary<string, string>(StringComparer.Ordinal) + { + ["suse.id"] = dto.AdvisoryId + }, + Etag: "adv-1", + LastModified: DateTimeOffset.UtcNow, + GridFsId: ObjectId.Empty); + + var mapped = SuseMapper.Map(dto, document, DateTimeOffset.UtcNow); + + Assert.Equal(dto.AdvisoryId, mapped.AdvisoryKey); + var package = Assert.Single(mapped.AffectedPackages); + Assert.Equal(AffectedPackageTypes.Rpm, package.Type); + var range = Assert.Single(package.VersionRanges); + Assert.Equal("nevra", range.RangeKind); + Assert.NotNull(range.Primitives); + Assert.NotNull(range.Primitives!.Nevra); + Assert.NotNull(range.Primitives.Nevra!.Fixed); + Assert.Equal("openssl", range.Primitives.Nevra.Fixed!.Name); + Assert.Equal("SUSE Linux Enterprise Server 15 SP5", package.Platform); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs new file mode 100644 index 00000000..5d6cd78f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.Suse.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Class1.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Class1.cs deleted file mode 100644 index 4bf1a97f..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Class1.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Distro.Suse; - -public sealed class DistroSuseConnectorPlugin : IConnectorPlugin -{ - public string Name => "distro-suse"; - - public bool IsAvailable(IServiceProvider services) => true; - - public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); - - private sealed class StubConnector : IFeedConnector - { - public StubConnector(string sourceName) => SourceName = sourceName; - - public string SourceName { get; } - - public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - } -} - diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs new file mode 100644 index 00000000..203f24af --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs @@ -0,0 +1,86 @@ +using System; + +namespace StellaOps.Feedser.Source.Distro.Suse.Configuration; + +public sealed class SuseOptions +{ + public const string HttpClientName = "feedser.suse"; + + /// <summary> + /// CSV index enumerating CSAF advisories with their last modification timestamps. + /// </summary> + public Uri ChangesEndpoint { get; set; } = new("https://ftp.suse.com/pub/projects/security/csaf/changes.csv"); + + /// <summary> + /// Base URI where individual CSAF advisories reside (filename appended verbatim). + /// </summary> + public Uri AdvisoryBaseUri { get; set; } = new("https://ftp.suse.com/pub/projects/security/csaf/"); + + /// <summary> + /// Maximum advisories to fetch per run to bound backfill effort. + /// </summary> + public int MaxAdvisoriesPerFetch { get; set; } = 40; + + /// <summary> + /// Initial history window for first-time execution. + /// </summary> + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + /// <summary> + /// Overlap window applied when resuming to capture late edits. + /// </summary> + public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(3); + + /// <summary> + /// Optional delay between advisory detail fetches. + /// </summary> + public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero; + + /// <summary> + /// Custom user agent presented to SUSE endpoints. + /// </summary> + public string UserAgent { get; set; } = "StellaOps.Feedser.Suse/0.1 (+https://stella-ops.org)"; + + /// <summary> + /// Timeout override applied to HTTP requests (defaults to 60 seconds when unset). + /// </summary> + public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45); + + public void Validate() + { + if (ChangesEndpoint is null || !ChangesEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("SuseOptions.ChangesEndpoint must be an absolute URI."); + } + + if (AdvisoryBaseUri is null || !AdvisoryBaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("SuseOptions.AdvisoryBaseUri must be an absolute URI."); + } + + if (MaxAdvisoriesPerFetch <= 0 || MaxAdvisoriesPerFetch > 250) + { + throw new InvalidOperationException("MaxAdvisoriesPerFetch must be between 1 and 250."); + } + + if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365)) + { + throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days."); + } + + if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14)) + { + throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days."); + } + + if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes."); + } + + if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10)) + { + throw new InvalidOperationException("RequestDelay must be between 0 and 10 seconds."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs new file mode 100644 index 00000000..e565c12f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal sealed record SuseAdvisoryDto( + string AdvisoryId, + string Title, + string? Summary, + DateTimeOffset Published, + IReadOnlyList<string> CveIds, + IReadOnlyList<SusePackageStateDto> Packages, + IReadOnlyList<SuseReferenceDto> References); + +internal sealed record SusePackageStateDto( + string Package, + string Platform, + string? Architecture, + string CanonicalNevra, + string? IntroducedVersion, + string? FixedVersion, + string? LastAffectedVersion, + string Status); + +internal sealed record SuseReferenceDto( + string Url, + string? Kind, + string? Title); diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs new file mode 100644 index 00000000..0a6f1c6a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs @@ -0,0 +1,5 @@ +using System; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal sealed record SuseChangeRecord(string FileName, DateTimeOffset ModifiedAt); diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs new file mode 100644 index 00000000..f5ccd4ac --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal static class SuseChangesParser +{ + public static IReadOnlyList<SuseChangeRecord> Parse(string csv) + { + if (string.IsNullOrWhiteSpace(csv)) + { + return Array.Empty<SuseChangeRecord>(); + } + + var records = new List<SuseChangeRecord>(); + using var reader = new StringReader(csv); + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parts = SplitCsvLine(line); + if (parts.Length < 2) + { + continue; + } + + var fileName = parts[0].Trim(); + if (string.IsNullOrWhiteSpace(fileName)) + { + continue; + } + + if (!DateTimeOffset.TryParse(parts[1], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var modifiedAt)) + { + continue; + } + + records.Add(new SuseChangeRecord(fileName, modifiedAt.ToUniversalTime())); + } + + return records; + } + + private static string[] SplitCsvLine(string line) + { + var values = new List<string>(2); + var current = string.Empty; + var insideQuotes = false; + + foreach (var ch in line) + { + if (ch == '"') + { + insideQuotes = !insideQuotes; + continue; + } + + if (ch == ',' && !insideQuotes) + { + values.Add(current); + current = string.Empty; + continue; + } + + current += ch; + } + + if (!string.IsNullOrEmpty(current)) + { + values.Add(current); + } + + return values.ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs new file mode 100644 index 00000000..649796fd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs @@ -0,0 +1,422 @@ +using System; +using System.Buffers.Text; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal static class SuseCsafParser +{ + public static SuseAdvisoryDto Parse(string json) + { + ArgumentException.ThrowIfNullOrEmpty(json); + + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + + if (!root.TryGetProperty("document", out var documentElement)) + { + throw new InvalidOperationException("CSAF payload missing 'document' element."); + } + + var trackingElement = documentElement.GetProperty("tracking"); + var advisoryId = trackingElement.TryGetProperty("id", out var idElement) + ? idElement.GetString() + : null; + if (string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("CSAF payload missing tracking.id."); + } + + var title = documentElement.TryGetProperty("title", out var titleElement) + ? titleElement.GetString() + : advisoryId; + + var summary = ExtractSummary(documentElement); + var published = ParseDate(trackingElement, "initial_release_date") + ?? ParseDate(trackingElement, "current_release_date") + ?? DateTimeOffset.UtcNow; + + var references = new List<SuseReferenceDto>(); + if (documentElement.TryGetProperty("references", out var referencesElement) && + referencesElement.ValueKind == JsonValueKind.Array) + { + foreach (var referenceElement in referencesElement.EnumerateArray()) + { + var url = referenceElement.TryGetProperty("url", out var urlElement) + ? urlElement.GetString() + : null; + + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + references.Add(new SuseReferenceDto( + url.Trim(), + referenceElement.TryGetProperty("category", out var categoryElement) ? categoryElement.GetString() : null, + referenceElement.TryGetProperty("summary", out var summaryElement) ? summaryElement.GetString() : null)); + } + } + + var productLookup = BuildProductLookup(root); + var packageBuilders = new Dictionary<string, PackageStateBuilder>(StringComparer.OrdinalIgnoreCase); + var cveIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + if (root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) && + vulnerabilitiesElement.ValueKind == JsonValueKind.Array) + { + foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) + { + if (vulnerability.TryGetProperty("cve", out var cveElement)) + { + var cve = cveElement.GetString(); + if (!string.IsNullOrWhiteSpace(cve)) + { + cveIds.Add(cve.Trim()); + } + } + + if (vulnerability.TryGetProperty("references", out var vulnReferences) && + vulnReferences.ValueKind == JsonValueKind.Array) + { + foreach (var referenceElement in vulnReferences.EnumerateArray()) + { + var url = referenceElement.TryGetProperty("url", out var urlElement) + ? urlElement.GetString() + : null; + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + references.Add(new SuseReferenceDto( + url.Trim(), + referenceElement.TryGetProperty("category", out var categoryElement) ? categoryElement.GetString() : null, + referenceElement.TryGetProperty("summary", out var summaryElement) ? summaryElement.GetString() : null)); + } + } + + if (!vulnerability.TryGetProperty("product_status", out var statusElement) || + statusElement.ValueKind != JsonValueKind.Object) + { + continue; + } + + foreach (var property in statusElement.EnumerateObject()) + { + var category = property.Name; + var idArray = property.Value; + if (idArray.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var productIdElement in idArray.EnumerateArray()) + { + var productId = productIdElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + if (!productLookup.TryGetValue(productId, out var product)) + { + continue; + } + + if (!packageBuilders.TryGetValue(productId, out var builder)) + { + builder = new PackageStateBuilder(product); + packageBuilders[productId] = builder; + } + + builder.ApplyStatus(category, product); + } + } + } + } + + var packages = new List<SusePackageStateDto>(packageBuilders.Count); + foreach (var builder in packageBuilders.Values) + { + if (builder.ShouldEmit) + { + packages.Add(builder.ToDto()); + } + } + + packages.Sort(static (left, right) => + { + var compare = string.Compare(left.Platform, right.Platform, StringComparison.OrdinalIgnoreCase); + if (compare != 0) + { + return compare; + } + + compare = string.Compare(left.Package, right.Package, StringComparison.OrdinalIgnoreCase); + if (compare != 0) + { + return compare; + } + + return string.Compare(left.Architecture, right.Architecture, StringComparison.OrdinalIgnoreCase); + }); + + var cveList = cveIds.Count == 0 + ? Array.Empty<string>() + : cveIds.OrderBy(static cve => cve, StringComparer.OrdinalIgnoreCase).ToArray(); + + return new SuseAdvisoryDto( + advisoryId.Trim(), + string.IsNullOrWhiteSpace(title) ? advisoryId : title!, + summary, + published, + cveList, + packages, + references); + } + + private static string? ExtractSummary(JsonElement documentElement) + { + if (!documentElement.TryGetProperty("notes", out var notesElement) || notesElement.ValueKind != JsonValueKind.Array) + { + return null; + } + + foreach (var note in notesElement.EnumerateArray()) + { + var category = note.TryGetProperty("category", out var categoryElement) + ? categoryElement.GetString() + : null; + + if (string.Equals(category, "summary", StringComparison.OrdinalIgnoreCase) + || string.Equals(category, "description", StringComparison.OrdinalIgnoreCase)) + { + return note.TryGetProperty("text", out var textElement) ? textElement.GetString() : null; + } + } + + return null; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var dateElement)) + { + return null; + } + + if (dateElement.ValueKind == JsonValueKind.String && + DateTimeOffset.TryParse(dateElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)) + { + return parsed.ToUniversalTime(); + } + + return null; + } + + private static Dictionary<string, SuseProduct> BuildProductLookup(JsonElement root) + { + var lookup = new Dictionary<string, SuseProduct>(StringComparer.OrdinalIgnoreCase); + + if (!root.TryGetProperty("product_tree", out var productTree)) + { + return lookup; + } + + if (productTree.TryGetProperty("branches", out var branches) && branches.ValueKind == JsonValueKind.Array) + { + TraverseBranches(branches, null, null, lookup); + } + + return lookup; + } + + private static void TraverseBranches(JsonElement branches, string? platform, string? architecture, IDictionary<string, SuseProduct> lookup) + { + foreach (var branch in branches.EnumerateArray()) + { + var category = branch.TryGetProperty("category", out var categoryElement) + ? categoryElement.GetString() + : null; + + var name = branch.TryGetProperty("name", out var nameElement) + ? nameElement.GetString() + : null; + + var nextPlatform = platform; + var nextArchitecture = architecture; + + if (string.Equals(category, "product_family", StringComparison.OrdinalIgnoreCase) || + string.Equals(category, "product_name", StringComparison.OrdinalIgnoreCase) || + string.Equals(category, "product_version", StringComparison.OrdinalIgnoreCase)) + { + if (!string.IsNullOrWhiteSpace(name)) + { + nextPlatform = name; + } + } + + if (string.Equals(category, "architecture", StringComparison.OrdinalIgnoreCase)) + { + nextArchitecture = string.IsNullOrWhiteSpace(name) ? null : name; + } + + if (branch.TryGetProperty("product", out var productElement) && productElement.ValueKind == JsonValueKind.Object) + { + var productId = productElement.TryGetProperty("product_id", out var idElement) + ? idElement.GetString() + : null; + + if (!string.IsNullOrWhiteSpace(productId)) + { + var productName = productElement.TryGetProperty("name", out var productNameElement) + ? productNameElement.GetString() + : productId; + + var (platformName, packageSegment) = SplitProductId(productId!, nextPlatform); + if (string.IsNullOrWhiteSpace(packageSegment)) + { + packageSegment = productName; + } + + if (string.IsNullOrWhiteSpace(packageSegment)) + { + continue; + } + + if (!Nevra.TryParse(packageSegment, out var nevra) && !Nevra.TryParse(productName ?? packageSegment, out nevra)) + { + continue; + } + + lookup[productId!] = new SuseProduct( + productId!, + platformName ?? "SUSE", + nevra!, + nextArchitecture ?? nevra!.Architecture); + } + } + + if (branch.TryGetProperty("branches", out var childBranches) && childBranches.ValueKind == JsonValueKind.Array) + { + TraverseBranches(childBranches, nextPlatform, nextArchitecture, lookup); + } + } + } + + private static (string? Platform, string? Package) SplitProductId(string productId, string? currentPlatform) + { + var separatorIndex = productId.IndexOf(':'); + if (separatorIndex < 0) + { + return (currentPlatform, productId); + } + + var platform = productId[..separatorIndex]; + var package = separatorIndex < productId.Length - 1 ? productId[(separatorIndex + 1)..] : string.Empty; + var platformNormalized = string.IsNullOrWhiteSpace(platform) ? currentPlatform : platform; + var packageNormalized = string.IsNullOrWhiteSpace(package) ? null : package; + return (platformNormalized, packageNormalized); + } + + private static string FormatNevraVersion(Nevra nevra) + { + var epochSegment = nevra.HasExplicitEpoch || nevra.Epoch > 0 ? $"{nevra.Epoch}:" : string.Empty; + return $"{epochSegment}{nevra.Version}-{nevra.Release}"; + } + + private sealed record SuseProduct(string ProductId, string Platform, Nevra Nevra, string? Architecture) + { + public string Package => Nevra.Name; + + public string Version => FormatNevraVersion(Nevra); + + public string CanonicalNevra => Nevra.ToCanonicalString(); + } + + private sealed class PackageStateBuilder + { + private readonly SuseProduct _product; + + public PackageStateBuilder(SuseProduct product) + { + _product = product; + Status = null; + } + + public string Package => _product.Package; + public string Platform => _product.Platform; + public string? Architecture => _product.Architecture; + public string? IntroducedVersion { get; private set; } + public string? FixedVersion { get; private set; } + public string? LastAffectedVersion { get; private set; } + public string? Status { get; private set; } + + public bool ShouldEmit => !string.IsNullOrWhiteSpace(Status) && !string.Equals(Status, "not_affected", StringComparison.OrdinalIgnoreCase); + + public void ApplyStatus(string category, SuseProduct product) + { + if (string.IsNullOrWhiteSpace(category)) + { + return; + } + + switch (category.ToLowerInvariant()) + { + case "recommended": + case "fixed": + FixedVersion = product.Version; + Status = "resolved"; + break; + + case "known_affected": + case "known_vulnerable": + LastAffectedVersion = product.Version; + Status ??= "open"; + break; + + case "first_affected": + IntroducedVersion ??= product.Version; + Status ??= "open"; + break; + + case "under_investigation": + Status ??= "investigating"; + break; + + case "known_not_affected": + Status = "not_affected"; + IntroducedVersion = null; + FixedVersion = null; + LastAffectedVersion = null; + break; + } + } + + public SusePackageStateDto ToDto() + { + var status = Status ?? "unknown"; + var introduced = IntroducedVersion; + var lastAffected = LastAffectedVersion; + + if (string.Equals(status, "resolved", StringComparison.OrdinalIgnoreCase) && string.IsNullOrWhiteSpace(FixedVersion)) + { + status = "open"; + } + + return new SusePackageStateDto( + Package, + Platform, + Architecture, + _product.CanonicalNevra, + introduced, + FixedVersion, + lastAffected, + status); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs new file mode 100644 index 00000000..38822801 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal sealed record SuseCursor( + DateTimeOffset? LastModified, + IReadOnlyCollection<string> ProcessedIds, + IReadOnlyCollection<Guid> PendingDocuments, + IReadOnlyCollection<Guid> PendingMappings, + IReadOnlyDictionary<string, SuseFetchCacheEntry> FetchCache) +{ + private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>(); + private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); + private static readonly IReadOnlyDictionary<string, SuseFetchCacheEntry> EmptyCache = + new Dictionary<string, SuseFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); + + public static SuseCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyCache); + + public static SuseCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastModified = null; + if (document.TryGetValue("lastModified", out var lastValue)) + { + lastModified = lastValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + var processed = ReadStringSet(document, "processedIds"); + var pendingDocs = ReadGuidSet(document, "pendingDocuments"); + var pendingMappings = ReadGuidSet(document, "pendingMappings"); + var cache = ReadCache(document); + + return new SuseCursor(lastModified, processed, pendingDocs, pendingMappings, cache); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), + }; + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + if (ProcessedIds.Count > 0) + { + document["processedIds"] = new BsonArray(ProcessedIds); + } + + if (FetchCache.Count > 0) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBsonDocument(); + } + + document["fetchCache"] = cacheDocument; + } + + return document; + } + + public SuseCursor WithPendingDocuments(IEnumerable<Guid> ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public SuseCursor WithPendingMappings(IEnumerable<Guid> ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public SuseCursor WithFetchCache(IDictionary<string, SuseFetchCacheEntry>? cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyCache }; + } + + return this with { FetchCache = new Dictionary<string, SuseFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public SuseCursor WithProcessed(DateTimeOffset modified, IEnumerable<string> ids) + => this with + { + LastModified = modified.ToUniversalTime(), + ProcessedIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyStringList + }; + + public bool TryGetCache(string key, out SuseFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = SuseFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + + private static IReadOnlyCollection<string> ReadStringSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyStringList; + } + + var list = new List<string>(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String) + { + var str = element.AsString.Trim(); + if (!string.IsNullOrWhiteSpace(str)) + { + list.Add(str); + } + } + } + + return list; + } + + private static IReadOnlyCollection<Guid> ReadGuidSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List<Guid>(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary<string, SuseFetchCacheEntry> ReadCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) + { + return EmptyCache; + } + + var cache = new Dictionary<string, SuseFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entry) + { + cache[element.Name] = SuseFetchCacheEntry.FromBson(entry); + } + } + + return cache; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs new file mode 100644 index 00000000..7d16c44a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs @@ -0,0 +1,76 @@ +using System; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal sealed record SuseFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) +{ + public static SuseFetchCacheEntry Empty { get; } = new(null, null); + + public static SuseFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + => new(document.Etag, document.LastModified); + + public static SuseFetchCacheEntry FromBson(BsonDocument document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + string? etag = null; + DateTimeOffset? lastModified = null; + + if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String) + { + etag = etagValue.AsString; + } + + if (document.TryGetValue("lastModified", out var modifiedValue)) + { + lastModified = modifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new SuseFetchCacheEntry(etag, lastModified); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (!string.IsNullOrWhiteSpace(ETag)) + { + document["etag"] = ETag; + } + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + return document; + } + + public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + { + if (document is null) + { + return false; + } + + if (!string.Equals(ETag, document.Etag, StringComparison.Ordinal)) + { + return false; + } + + if (LastModified.HasValue && document.LastModified.HasValue) + { + return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime; + } + + return !LastModified.HasValue && !document.LastModified.HasValue; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs new file mode 100644 index 00000000..2d19635f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Distro.Suse.Internal; + +internal static class SuseMapper +{ + public static Advisory Map(SuseAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var packages = BuildPackages(dto, recordedAt); + + var fetchProvenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mapProvenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "mapping", + dto.AdvisoryId, + recordedAt); + + var published = dto.Published; + var modified = DateTimeOffset.Compare(recordedAt, dto.Published) >= 0 ? recordedAt : dto.Published; + + return new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title ?? dto.AdvisoryId, + summary: dto.Summary, + language: "en", + published: published, + modified: modified, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: Array.Empty<CvssMetric>(), + provenance: new[] { fetchProvenance, mapProvenance }); + } + + private static string[] BuildAliases(SuseAdvisoryDto dto) + { + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId + }; + + foreach (var cve in dto.CveIds ?? Array.Empty<string>()) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve.Trim()); + } + } + + return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static AdvisoryReference[] BuildReferences(SuseAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty<AdvisoryReference>(); + } + + var references = new List<AdvisoryReference>(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + try + { + var provenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt); + + references.Add(new AdvisoryReference( + reference.Url.Trim(), + NormalizeReferenceKind(reference.Kind), + reference.Kind, + reference.Title, + provenance)); + } + catch (ArgumentException) + { + // Ignore malformed URLs to keep advisory mapping resilient. + } + } + + return references.Count == 0 + ? Array.Empty<AdvisoryReference>() + : references + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeReferenceKind(string? kind) + { + if (string.IsNullOrWhiteSpace(kind)) + { + return null; + } + + return kind.Trim().ToLowerInvariant() switch + { + "cve" => "cve", + "self" => "advisory", + "external" => "external", + _ => null, + }; + } + + private static IReadOnlyList<AffectedPackage> BuildPackages(SuseAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.Packages is null || dto.Packages.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + var packages = new List<AffectedPackage>(dto.Packages.Count); + foreach (var package in dto.Packages) + { + if (string.IsNullOrWhiteSpace(package.CanonicalNevra)) + { + continue; + } + + Nevra? nevra; + if (!Nevra.TryParse(package.CanonicalNevra, out nevra)) + { + continue; + } + + var affectedProvenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "affected", + $"{package.Platform}:{package.CanonicalNevra}", + recordedAt); + + var ranges = BuildVersionRanges(package, nevra!, recordedAt); + if (ranges.Count == 0 && string.Equals(package.Status, "not_affected", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Rpm, + identifier: nevra!.ToCanonicalString(), + platform: package.Platform, + versionRanges: ranges, + statuses: BuildStatuses(package, affectedProvenance), + provenance: new[] { affectedProvenance })); + } + + return packages.Count == 0 + ? Array.Empty<AffectedPackage>() + : packages + .OrderBy(static pkg => pkg.Platform, StringComparer.OrdinalIgnoreCase) + .ThenBy(static pkg => pkg.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList<AffectedPackageStatus> BuildStatuses(SusePackageStateDto package, AdvisoryProvenance provenance) + { + if (string.IsNullOrWhiteSpace(package.Status)) + { + return Array.Empty<AffectedPackageStatus>(); + } + + return new[] + { + new AffectedPackageStatus(package.Status, provenance) + }; + } + + private static IReadOnlyList<AffectedVersionRange> BuildVersionRanges(SusePackageStateDto package, Nevra nevra, DateTimeOffset recordedAt) + { + var introducedComponent = ParseNevraComponent(package.IntroducedVersion, nevra); + var fixedComponent = ParseNevraComponent(package.FixedVersion, nevra); + var lastAffectedComponent = ParseNevraComponent(package.LastAffectedVersion, nevra); + + if (introducedComponent is null && fixedComponent is null && lastAffectedComponent is null) + { + return Array.Empty<AffectedVersionRange>(); + } + + var rangeProvenance = new AdvisoryProvenance( + SuseConnectorPlugin.SourceName, + "range", + $"{package.Platform}:{nevra.ToCanonicalString()}", + recordedAt); + + var extensions = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["suse.status"] = package.Status + }; + + var rangeExpression = BuildRangeExpression(package.IntroducedVersion, package.FixedVersion, package.LastAffectedVersion); + + var range = new AffectedVersionRange( + rangeKind: "nevra", + introducedVersion: package.IntroducedVersion, + fixedVersion: package.FixedVersion, + lastAffectedVersion: package.LastAffectedVersion, + rangeExpression: rangeExpression, + provenance: rangeProvenance, + primitives: new RangePrimitives( + SemVer: null, + Nevra: new NevraPrimitive(introducedComponent, fixedComponent, lastAffectedComponent), + Evr: null, + VendorExtensions: extensions)); + + return new[] { range }; + } + + private static NevraComponent? ParseNevraComponent(string? version, Nevra nevra) + { + if (string.IsNullOrWhiteSpace(version)) + { + return null; + } + + if (!TrySplitNevraVersion(version.Trim(), out var epoch, out var ver, out var rel)) + { + return null; + } + + return new NevraComponent( + nevra.Name, + epoch, + ver, + rel, + string.IsNullOrWhiteSpace(nevra.Architecture) ? null : nevra.Architecture); + } + + private static bool TrySplitNevraVersion(string value, out int epoch, out string version, out string release) + { + epoch = 0; + version = string.Empty; + release = string.Empty; + + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var dashIndex = trimmed.LastIndexOf('-'); + if (dashIndex <= 0 || dashIndex >= trimmed.Length - 1) + { + return false; + } + + release = trimmed[(dashIndex + 1)..]; + var versionSegment = trimmed[..dashIndex]; + + var epochIndex = versionSegment.IndexOf(':'); + if (epochIndex >= 0) + { + var epochPart = versionSegment[..epochIndex]; + version = epochIndex < versionSegment.Length - 1 ? versionSegment[(epochIndex + 1)..] : string.Empty; + if (epochPart.Length > 0 && !int.TryParse(epochPart, NumberStyles.Integer, CultureInfo.InvariantCulture, out epoch)) + { + epoch = 0; + return false; + } + } + else + { + version = versionSegment; + } + + return !string.IsNullOrWhiteSpace(version) && !string.IsNullOrWhiteSpace(release); + } + + private static string? BuildRangeExpression(string? introduced, string? fixedVersion, string? lastAffected) + { + var parts = new List<string>(3); + if (!string.IsNullOrWhiteSpace(introduced)) + { + parts.Add($"introduced:{introduced}"); + } + + if (!string.IsNullOrWhiteSpace(fixedVersion)) + { + parts.Add($"fixed:{fixedVersion}"); + } + + if (!string.IsNullOrWhiteSpace(lastAffected)) + { + parts.Add($"last:{lastAffected}"); + } + + return parts.Count == 0 ? null : string.Join(" ", parts); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs new file mode 100644 index 00000000..c138995c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +internal static class SuseJobKinds +{ + public const string Fetch = "source:suse:fetch"; + public const string Parse = "source:suse:parse"; + public const string Map = "source:suse:map"; +} + +internal sealed class SuseFetchJob : IJob +{ + private readonly SuseConnector _connector; + + public SuseFetchJob(SuseConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class SuseParseJob : IJob +{ + private readonly SuseConnector _connector; + + public SuseParseJob(SuseConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class SuseMapJob : IJob +{ + private readonly SuseConnector _connector; + + public SuseMapJob(SuseConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj b/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj index 182529d4..34c6b8e9 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj +++ b/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj @@ -11,6 +11,7 @@ <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> </ItemGroup> </Project> - diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs b/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs new file mode 100644 index 00000000..4f897f29 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs @@ -0,0 +1,573 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Distro.Suse.Configuration; +using StellaOps.Feedser.Source.Distro.Suse.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public sealed class SuseConnector : IFeedConnector +{ + private static readonly Action<ILogger, string, int, Exception?> LogMapped = + LoggerMessage.Define<string, int>( + LogLevel.Information, + new EventId(1, "SuseMapped"), + "SUSE advisory {AdvisoryId} mapped with {AffectedCount} affected packages"); + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly SuseOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger<SuseConnector> _logger; + + public SuseConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions<SuseOptions> options, + TimeProvider? timeProvider, + ILogger<SuseConnector> logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => SuseConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments); + var pendingMappings = new HashSet<Guid>(cursor.PendingMappings); + var fetchCache = new Dictionary<string, SuseFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var touchedResources = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + var changesUri = _options.ChangesEndpoint; + var changesKey = changesUri.ToString(); + touchedResources.Add(changesKey); + + cursor.TryGetCache(changesKey, out var cachedChanges); + + var changesRequest = new SourceFetchRequest(SuseOptions.HttpClientName, SourceName, changesUri) + { + Metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["suse.type"] = "changes" + }, + AcceptHeaders = new[] { "text/csv", "text/plain" }, + TimeoutOverride = _options.FetchTimeout, + ETag = cachedChanges?.ETag, + LastModified = cachedChanges?.LastModified, + }; + + SourceFetchResult changesResult; + try + { + changesResult = await _fetchService.FetchAsync(changesRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "SUSE changes.csv fetch failed from {Uri}", changesUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var maxModified = cursor.LastModified ?? DateTimeOffset.MinValue; + var processedUpdated = false; + var processedIds = new HashSet<string>(cursor.ProcessedIds, StringComparer.OrdinalIgnoreCase); + var currentWindowIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + IReadOnlyList<SuseChangeRecord> changeRecords = Array.Empty<SuseChangeRecord>(); + if (changesResult.IsNotModified) + { + if (cursor.FetchCache.TryGetValue(changesKey, out var existingCache)) + { + fetchCache[changesKey] = existingCache; + } + } + else if (changesResult.IsSuccess && changesResult.Document is not null) + { + fetchCache[changesKey] = SuseFetchCacheEntry.FromDocument(changesResult.Document); + if (changesResult.Document.GridFsId.HasValue) + { + byte[] changesBytes; + try + { + changesBytes = await _rawDocumentStorage.DownloadAsync(changesResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download SUSE changes.csv document {DocumentId}", changesResult.Document.Id); + throw; + } + + var csv = Encoding.UTF8.GetString(changesBytes); + changeRecords = SuseChangesParser.Parse(csv); + } + } + + if (changeRecords.Count > 0) + { + var baseline = (cursor.LastModified ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap; + if (baseline < DateTimeOffset.UnixEpoch) + { + baseline = DateTimeOffset.UnixEpoch; + } + + ProvenanceDiagnostics.ReportResumeWindow(SourceName, baseline, _logger); + + var candidates = changeRecords + .Where(record => record.ModifiedAt >= baseline) + .OrderBy(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (candidates.Count == 0) + { + candidates = changeRecords + .OrderByDescending(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxAdvisoriesPerFetch) + .OrderBy(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + else if (candidates.Count > _options.MaxAdvisoriesPerFetch) + { + candidates = candidates + .OrderByDescending(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxAdvisoriesPerFetch) + .OrderBy(record => record.ModifiedAt) + .ThenBy(record => record.FileName, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + foreach (var record in candidates) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = new Uri(_options.AdvisoryBaseUri, record.FileName); + var cacheKey = detailUri.AbsoluteUri; + touchedResources.Add(cacheKey); + + cursor.TryGetCache(cacheKey, out var cachedEntry); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); + + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["suse.file"] = record.FileName, + ["suse.modified"] = record.ModifiedAt.ToString("O", CultureInfo.InvariantCulture) + }; + + if (!metadata.ContainsKey("suse.id") && existing?.Metadata?.TryGetValue("suse.id", out var existingId) == true) + { + metadata["suse.id"] = existingId; + } + + var request = new SourceFetchRequest(SuseOptions.HttpClientName, SourceName, detailUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "application/json", "text/json" }, + TimeoutOverride = _options.FetchTimeout, + ETag = existing?.Etag ?? cachedEntry?.ETag, + LastModified = existing?.LastModified ?? cachedEntry?.LastModified, + }; + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to fetch SUSE advisory {FileName}", record.FileName); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.IsNotModified) + { + if (existing is not null) + { + fetchCache[cacheKey] = SuseFetchCacheEntry.FromDocument(existing); + if (string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + pendingDocuments.Remove(existing.Id); + pendingMappings.Remove(existing.Id); + } + } + + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + fetchCache[cacheKey] = SuseFetchCacheEntry.FromDocument(result.Document); + pendingDocuments.Add(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + currentWindowIds.Add(record.FileName); + + if (record.ModifiedAt > maxModified) + { + maxModified = record.ModifiedAt; + processedUpdated = true; + } + } + } + + if (fetchCache.Count > 0 && touchedResources.Count > 0) + { + var staleKeys = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in staleKeys) + { + fetchCache.Remove(key); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithFetchCache(fetchCache); + + if (processedUpdated && currentWindowIds.Count > 0) + { + updatedCursor = updatedCursor.WithProcessed(maxModified, currentWindowIds); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remaining = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remaining.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("SUSE document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download SUSE document {DocumentId}", document.Id); + throw; + } + + SuseAdvisoryDto dto; + try + { + var json = Encoding.UTF8.GetString(bytes); + dto = SuseCsafParser.Parse(json); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse SUSE advisory {Uri}", document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + var metadata = document.Metadata is null + ? new Dictionary<string, string>(StringComparer.Ordinal) + : new Dictionary<string, string>(document.Metadata, StringComparer.Ordinal); + + metadata["suse.id"] = dto.AdvisoryId; + var updatedDocument = document with { Metadata = metadata }; + await _documentStore.UpsertAsync(updatedDocument, cancellationToken).ConfigureAwait(false); + + var payload = ToBson(dto); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "suse.csaf.v1", payload, _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remaining.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remaining) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + SuseAdvisoryDto dto; + try + { + dto = FromBson(dtoRecord.Payload); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize SUSE DTO for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = SuseMapper.Map(dto, document, _timeProvider.GetUtcNow()); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + + LogMapped(_logger, dto.AdvisoryId, advisory.AffectedPackages.Length, null); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task<SuseCursor> GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? SuseCursor.Empty : SuseCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(SuseCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static BsonDocument ToBson(SuseAdvisoryDto dto) + { + var packages = new BsonArray(); + foreach (var package in dto.Packages) + { + var packageDoc = new BsonDocument + { + ["package"] = package.Package, + ["platform"] = package.Platform, + ["canonical"] = package.CanonicalNevra, + ["status"] = package.Status + }; + + if (!string.IsNullOrWhiteSpace(package.Architecture)) + { + packageDoc["arch"] = package.Architecture; + } + + if (!string.IsNullOrWhiteSpace(package.IntroducedVersion)) + { + packageDoc["introduced"] = package.IntroducedVersion; + } + + if (!string.IsNullOrWhiteSpace(package.FixedVersion)) + { + packageDoc["fixed"] = package.FixedVersion; + } + + if (!string.IsNullOrWhiteSpace(package.LastAffectedVersion)) + { + packageDoc["last"] = package.LastAffectedVersion; + } + + packages.Add(packageDoc); + } + + var references = new BsonArray(); + foreach (var reference in dto.References) + { + var referenceDoc = new BsonDocument + { + ["url"] = reference.Url + }; + + if (!string.IsNullOrWhiteSpace(reference.Kind)) + { + referenceDoc["kind"] = reference.Kind; + } + + if (!string.IsNullOrWhiteSpace(reference.Title)) + { + referenceDoc["title"] = reference.Title; + } + + references.Add(referenceDoc); + } + + return new BsonDocument + { + ["advisoryId"] = dto.AdvisoryId, + ["title"] = dto.Title ?? string.Empty, + ["summary"] = dto.Summary ?? string.Empty, + ["published"] = dto.Published.UtcDateTime, + ["cves"] = new BsonArray(dto.CveIds ?? Array.Empty<string>()), + ["packages"] = packages, + ["references"] = references + }; + } + + private static SuseAdvisoryDto FromBson(BsonDocument document) + { + var advisoryId = document.GetValue("advisoryId", string.Empty).AsString; + var title = document.GetValue("title", advisoryId).AsString; + var summary = document.TryGetValue("summary", out var summaryValue) ? summaryValue.AsString : null; + var published = document.TryGetValue("published", out var publishedValue) + ? publishedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => DateTimeOffset.UtcNow + } + : DateTimeOffset.UtcNow; + + var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray bsonCves + ? bsonCves.OfType<BsonValue>() + .Select(static value => value?.ToString()) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value!) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() + : Array.Empty<string>(); + + var packageList = new List<SusePackageStateDto>(); + if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray bsonPackages) + { + foreach (var element in bsonPackages.OfType<BsonDocument>()) + { + var package = element.GetValue("package", string.Empty).AsString; + var platform = element.GetValue("platform", string.Empty).AsString; + var canonical = element.GetValue("canonical", string.Empty).AsString; + var status = element.GetValue("status", "unknown").AsString; + + var architecture = element.TryGetValue("arch", out var archValue) ? archValue.AsString : null; + var introduced = element.TryGetValue("introduced", out var introducedValue) ? introducedValue.AsString : null; + var fixedVersion = element.TryGetValue("fixed", out var fixedValue) ? fixedValue.AsString : null; + var last = element.TryGetValue("last", out var lastValue) ? lastValue.AsString : null; + + packageList.Add(new SusePackageStateDto( + package, + platform, + architecture, + canonical, + introduced, + fixedVersion, + last, + status)); + } + } + + var referenceList = new List<SuseReferenceDto>(); + if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray bsonReferences) + { + foreach (var element in bsonReferences.OfType<BsonDocument>()) + { + var url = element.GetValue("url", string.Empty).AsString; + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + referenceList.Add(new SuseReferenceDto( + url, + element.TryGetValue("kind", out var kindValue) ? kindValue.AsString : null, + element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null)); + } + } + + return new SuseAdvisoryDto( + advisoryId, + string.IsNullOrWhiteSpace(title) ? advisoryId : title, + string.IsNullOrWhiteSpace(summary) ? null : summary, + published, + cves, + packageList, + referenceList); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs new file mode 100644 index 00000000..76b74412 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public sealed class SuseConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "distro-suse"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance<SuseConnector>(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs new file mode 100644 index 00000000..ac7445e0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Distro.Suse.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public sealed class SuseDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:suse"; + private const string FetchCron = "*/30 * * * *"; + private const string ParseCron = "5,35 * * * *"; + private const string MapCron = "10,40 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddSuseConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob<SuseFetchJob>( + SuseJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob<SuseParseJob>( + SuseJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob<SuseMapJob>( + SuseJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs new file mode 100644 index 00000000..17aaf4c8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs @@ -0,0 +1,35 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Distro.Suse.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public static class SuseServiceCollectionExtensions +{ + public static IServiceCollection AddSuseConnector(this IServiceCollection services, Action<SuseOptions> configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions<SuseOptions>() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(SuseOptions.HttpClientName, (sp, httpOptions) => + { + var options = sp.GetRequiredService<IOptions<SuseOptions>>().Value; + httpOptions.BaseAddress = new Uri(options.AdvisoryBaseUri.GetLeftPart(UriPartial.Authority), UriKind.Absolute); + httpOptions.Timeout = options.FetchTimeout; + httpOptions.UserAgent = options.UserAgent; + httpOptions.AllowedHosts.Clear(); + httpOptions.AllowedHosts.Add(options.AdvisoryBaseUri.Host); + httpOptions.AllowedHosts.Add(options.ChangesEndpoint.Host); + httpOptions.DefaultRequestHeaders["Accept"] = "text/csv,application/json;q=0.9,text/plain;q=0.8"; + }); + + services.AddTransient<SuseConnector>(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json new file mode 100644 index 00000000..a1a9ee51 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json @@ -0,0 +1,40 @@ +{ + "offset": 0, + "limit": 1, + "total_results": 2, + "notices": [ + { + "id": "USN-9001-1", + "title": "Kernel update", + "summary": "Kernel fixes", + "published": "2025-01-20T08:30:00Z", + "cves_ids": [ + "CVE-2025-2000" + ], + "cves": [ + { + "id": "CVE-2025-2000" + } + ], + "references": [], + "release_packages": { + "noble": [ + { + "name": "linux-image", + "version": "6.8.0-1010.11", + "pocket": "security", + "is_source": false + } + ], + "focal": [ + { + "name": "linux-image", + "version": "5.15.0-200.0", + "pocket": "esm-infra", + "is_source": false + } + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json new file mode 100644 index 00000000..e7a8ef05 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json @@ -0,0 +1,42 @@ +{ + "offset": 1, + "limit": 1, + "total_results": 2, + "notices": [ + { + "id": "USN-9000-1", + "title": "Example security update", + "summary": "Package fixes", + "published": "2025-01-15T12:00:00Z", + "cves_ids": [ + "CVE-2025-1000", + "CVE-2025-1001" + ], + "cves": [ + { + "id": "CVE-2025-1000" + }, + { + "id": "CVE-2025-1001" + } + ], + "references": [ + { + "url": "https://ubuntu.com/security/USN-9000-1", + "category": "self", + "summary": "USN" + } + ], + "release_packages": { + "jammy": [ + { + "name": "examplepkg", + "version": "1.2.3-0ubuntu0.22.04.1", + "pocket": "security", + "is_source": false + } + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj new file mode 100644 index 00000000..c02f7b43 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Update="Fixtures\**\*"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs new file mode 100644 index 00000000..2222ab5a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs @@ -0,0 +1,171 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.Ubuntu; +using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Testing; +using Xunit; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Tests; + +[Collection("mongo-fixture")] +public sealed class UbuntuConnectorTests : IAsyncLifetime +{ + private static readonly Uri IndexPage0Uri = new("https://ubuntu.com/security/notices.json?offset=0&limit=1"); + private static readonly Uri IndexPage1Uri = new("https://ubuntu.com/security/notices.json?offset=1&limit=1"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public UbuntuConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 25, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_GeneratesEvrRangePrimitives() + { + await using var provider = await BuildServiceProviderAsync(); + + SeedInitialResponses(); + + var connector = provider.GetRequiredService<UbuntuConnector>(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var kernelNotice = advisories.Single(a => a.AdvisoryKey == "USN-9001-1"); + var noblePackage = Assert.Single(kernelNotice.AffectedPackages, pkg => pkg.Platform == "noble"); + var range = Assert.Single(noblePackage.VersionRanges); + Assert.Equal("evr", range.RangeKind); + Assert.NotNull(range.Primitives); + Assert.NotNull(range.Primitives!.Evr?.Fixed); + Assert.Contains("CVE-2025-2000", kernelNotice.Aliases); + + SeedNotModifiedResponses(); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + _handler.AssertNoPendingResponses(); + } + + private async Task<ServiceProvider> BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton<TimeProvider>(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddUbuntuConnector(options => + { + options.NoticesEndpoint = new Uri("https://ubuntu.com/security/notices.json"); + options.NoticeDetailBaseUri = new Uri("https://ubuntu.com/security/"); + options.MaxNoticesPerFetch = 2; + options.IndexPageSize = 1; + }); + + services.Configure<HttpClientFactoryOptions>(UbuntuOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + _handler.AddResponse(IndexPage0Uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/ubuntu-notices-page0.json"), Encoding.UTF8, "application/json") + }; + response.Headers.ETag = new EntityTagHeaderValue("\"index-page0-v1\""); + return response; + }); + + _handler.AddResponse(IndexPage1Uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/ubuntu-notices-page1.json"), Encoding.UTF8, "application/json") + }; + response.Headers.ETag = new EntityTagHeaderValue("\"index-page1-v1\""); + return response; + }); + } + + private void SeedNotModifiedResponses() + { + _handler.AddResponse(IndexPage0Uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue("\"index-page0-v1\""); + return response; + }); + + // Page 1 remains cached; the connector should skip fetching it when page 0 is unchanged. + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath.Replace('/', Path.DirectorySeparatorChar)); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Fixture '{relativePath}' not found.", path); + } + + return File.ReadAllText(path); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Class1.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Class1.cs deleted file mode 100644 index 26e52c3d..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Class1.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Distro.Ubuntu; - -public sealed class DistroUbuntuConnectorPlugin : IConnectorPlugin -{ - public string Name => "distro-ubuntu"; - - public bool IsAvailable(IServiceProvider services) => true; - - public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); - - private sealed class StubConnector : IFeedConnector - { - public StubConnector(string sourceName) => SourceName = sourceName; - - public string SourceName { get; } - - public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - - public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; - } -} - diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs new file mode 100644 index 00000000..26f05254 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs @@ -0,0 +1,69 @@ +using System; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; + +public sealed class UbuntuOptions +{ + public const string HttpClientName = "feedser.ubuntu"; + public const int MaxPageSize = 20; + + /// <summary> + /// Endpoint exposing the rolling JSON index of Ubuntu Security Notices. + /// </summary> + public Uri NoticesEndpoint { get; set; } = new("https://ubuntu.com/security/notices.json"); + + /// <summary> + /// Base URI where individual notice detail pages live. + /// </summary> + public Uri NoticeDetailBaseUri { get; set; } = new("https://ubuntu.com/security/"); + + public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(45); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan ResumeOverlap { get; set; } = TimeSpan.FromDays(3); + + public int MaxNoticesPerFetch { get; set; } = 60; + + public int IndexPageSize { get; set; } = 20; + + public string UserAgent { get; set; } = "StellaOps.Feedser.Ubuntu/0.1 (+https://stella-ops.org)"; + + public void Validate() + { + if (NoticesEndpoint is null || !NoticesEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("Ubuntu notices endpoint must be an absolute URI."); + } + + if (NoticeDetailBaseUri is null || !NoticeDetailBaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Ubuntu notice detail base URI must be an absolute URI."); + } + + if (MaxNoticesPerFetch <= 0 || MaxNoticesPerFetch > 200) + { + throw new InvalidOperationException("MaxNoticesPerFetch must be between 1 and 200."); + } + + if (FetchTimeout <= TimeSpan.Zero || FetchTimeout > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("FetchTimeout must be positive and less than five minutes."); + } + + if (InitialBackfill < TimeSpan.Zero || InitialBackfill > TimeSpan.FromDays(365)) + { + throw new InvalidOperationException("InitialBackfill must be between 0 and 365 days."); + } + + if (ResumeOverlap < TimeSpan.Zero || ResumeOverlap > TimeSpan.FromDays(14)) + { + throw new InvalidOperationException("ResumeOverlap must be between 0 and 14 days."); + } + + if (IndexPageSize <= 0 || IndexPageSize > MaxPageSize) + { + throw new InvalidOperationException($"IndexPageSize must be between 1 and {MaxPageSize}."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs new file mode 100644 index 00000000..4d1f607a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal sealed record UbuntuCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection<string> ProcessedNoticeIds, + IReadOnlyCollection<Guid> PendingDocuments, + IReadOnlyCollection<Guid> PendingMappings, + IReadOnlyDictionary<string, UbuntuFetchCacheEntry> FetchCache) +{ + private static readonly IReadOnlyCollection<string> EmptyIds = Array.Empty<string>(); + private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); + private static readonly IReadOnlyDictionary<string, UbuntuFetchCacheEntry> EmptyCache = + new Dictionary<string, UbuntuFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); + + public static UbuntuCursor Empty { get; } = new(null, EmptyIds, EmptyGuidList, EmptyGuidList, EmptyCache); + + public static UbuntuCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastPublished = null; + if (document.TryGetValue("lastPublished", out var value)) + { + lastPublished = value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null + }; + } + + var processed = ReadStringSet(document, "processedIds"); + var pendingDocuments = ReadGuidSet(document, "pendingDocuments"); + var pendingMappings = ReadGuidSet(document, "pendingMappings"); + var cache = ReadCache(document); + + return new UbuntuCursor(lastPublished, processed, pendingDocuments, pendingMappings, cache); + } + + public BsonDocument ToBsonDocument() + { + var doc = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())) + }; + + if (LastPublished.HasValue) + { + doc["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + if (ProcessedNoticeIds.Count > 0) + { + doc["processedIds"] = new BsonArray(ProcessedNoticeIds); + } + + if (FetchCache.Count > 0) + { + var cacheDoc = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDoc[key] = entry.ToBsonDocument(); + } + + doc["fetchCache"] = cacheDoc; + } + + return doc; + } + + public UbuntuCursor WithPendingDocuments(IEnumerable<Guid> ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public UbuntuCursor WithPendingMappings(IEnumerable<Guid> ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public UbuntuCursor WithFetchCache(IDictionary<string, UbuntuFetchCacheEntry>? cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyCache }; + } + + return this with { FetchCache = new Dictionary<string, UbuntuFetchCacheEntry>(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public UbuntuCursor WithProcessed(DateTimeOffset published, IEnumerable<string> ids) + => this with + { + LastPublished = published.ToUniversalTime(), + ProcessedNoticeIds = ids?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyIds + }; + + public bool TryGetCache(string key, out UbuntuFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = UbuntuFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + + private static IReadOnlyCollection<string> ReadStringSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyIds; + } + + var list = new List<string>(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String) + { + var str = element.AsString.Trim(); + if (!string.IsNullOrWhiteSpace(str)) + { + list.Add(str); + } + } + } + + return list; + } + + private static IReadOnlyCollection<Guid> ReadGuidSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List<Guid>(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary<string, UbuntuFetchCacheEntry> ReadCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDoc || cacheDoc.ElementCount == 0) + { + return EmptyCache; + } + + var cache = new Dictionary<string, UbuntuFetchCacheEntry>(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDoc.Elements) + { + if (element.Value is BsonDocument entryDoc) + { + cache[element.Name] = UbuntuFetchCacheEntry.FromBson(entryDoc); + } + } + + return cache; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs new file mode 100644 index 00000000..b4e4b261 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs @@ -0,0 +1,76 @@ +using System; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal sealed record UbuntuFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) +{ + public static UbuntuFetchCacheEntry Empty { get; } = new(null, null); + + public static UbuntuFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + => new(document.Etag, document.LastModified); + + public static UbuntuFetchCacheEntry FromBson(BsonDocument document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + string? etag = null; + DateTimeOffset? lastModified = null; + + if (document.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String) + { + etag = etagValue.AsString; + } + + if (document.TryGetValue("lastModified", out var modifiedValue)) + { + lastModified = modifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(modifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(modifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null + }; + } + + return new UbuntuFetchCacheEntry(etag, lastModified); + } + + public BsonDocument ToBsonDocument() + { + var doc = new BsonDocument(); + if (!string.IsNullOrWhiteSpace(ETag)) + { + doc["etag"] = ETag; + } + + if (LastModified.HasValue) + { + doc["lastModified"] = LastModified.Value.UtcDateTime; + } + + return doc; + } + + public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + { + if (document is null) + { + return false; + } + + if (!string.Equals(ETag, document.Etag, StringComparison.Ordinal)) + { + return false; + } + + if (LastModified.HasValue && document.LastModified.HasValue) + { + return LastModified.Value.UtcDateTime == document.LastModified.Value.UtcDateTime; + } + + return !LastModified.HasValue && !document.LastModified.HasValue; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs new file mode 100644 index 00000000..cab2e44e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs @@ -0,0 +1,217 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal static class UbuntuMapper +{ + public static Advisory Map(UbuntuNoticeDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var packages = BuildPackages(dto, recordedAt); + + var fetchProvenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mapProvenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "mapping", + dto.NoticeId, + recordedAt); + + return new Advisory( + advisoryKey: dto.NoticeId, + title: dto.Title ?? dto.NoticeId, + summary: dto.Summary, + language: "en", + published: dto.Published, + modified: recordedAt > dto.Published ? recordedAt : dto.Published, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: Array.Empty<CvssMetric>(), + provenance: new[] { fetchProvenance, mapProvenance }); + } + + private static string[] BuildAliases(UbuntuNoticeDto dto) + { + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + dto.NoticeId + }; + + foreach (var cve in dto.CveIds ?? Array.Empty<string>()) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve.Trim()); + } + } + + return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static AdvisoryReference[] BuildReferences(UbuntuNoticeDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty<AdvisoryReference>(); + } + + var references = new List<AdvisoryReference>(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + try + { + var provenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt); + + references.Add(new AdvisoryReference( + reference.Url.Trim(), + NormalizeReferenceKind(reference.Kind), + reference.Kind, + reference.Title, + provenance)); + } + catch (ArgumentException) + { + // ignore poorly formed URIs + } + } + + return references.Count == 0 + ? Array.Empty<AdvisoryReference>() + : references + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeReferenceKind(string? kind) + { + if (string.IsNullOrWhiteSpace(kind)) + { + return null; + } + + return kind.Trim().ToLowerInvariant() switch + { + "external" => "external", + "self" => "advisory", + _ => null + }; + } + + private static IReadOnlyList<AffectedPackage> BuildPackages(UbuntuNoticeDto dto, DateTimeOffset recordedAt) + { + if (dto.Packages is null || dto.Packages.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + var list = new List<AffectedPackage>(); + foreach (var package in dto.Packages) + { + if (string.IsNullOrWhiteSpace(package.Package) || string.IsNullOrWhiteSpace(package.Version)) + { + continue; + } + + if (!DebianEvr.TryParse(package.Version, out var evr) || evr is null) + { + continue; + } + + var provenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "affected", + $"{dto.NoticeId}:{package.Release}:{package.Package}", + recordedAt); + + var rangeProvenance = new AdvisoryProvenance( + UbuntuConnectorPlugin.SourceName, + "range", + $"{dto.NoticeId}:{package.Release}:{package.Package}", + recordedAt); + + var rangeExpression = $"fixed:{package.Version}"; + + var extensions = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["ubuntu.release"] = package.Release, + ["ubuntu.pocket"] = package.Pocket ?? string.Empty + }; + + var range = new AffectedVersionRange( + rangeKind: "evr", + introducedVersion: null, + fixedVersion: package.Version, + lastAffectedVersion: null, + rangeExpression: rangeExpression, + provenance: rangeProvenance, + primitives: new RangePrimitives( + SemVer: null, + Nevra: null, + Evr: new EvrPrimitive( + Introduced: null, + Fixed: new EvrComponent(evr.Epoch, evr.Version, evr.Revision.Length == 0 ? null : evr.Revision), + LastAffected: null), + VendorExtensions: extensions)); + + var statuses = new[] + { + new AffectedPackageStatus(DetermineStatus(package), provenance) + }; + + list.Add(new AffectedPackage( + type: AffectedPackageTypes.Deb, + identifier: package.Package, + platform: package.Release, + versionRanges: new[] { range }, + statuses: statuses, + provenance: new[] { provenance })); + } + + return list.Count == 0 + ? Array.Empty<AffectedPackage>() + : list + .OrderBy(static pkg => pkg.Platform, StringComparer.OrdinalIgnoreCase) + .ThenBy(static pkg => pkg.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string DetermineStatus(UbuntuReleasePackageDto package) + { + if (!string.IsNullOrWhiteSpace(package.Pocket) && package.Pocket.Contains("security", StringComparison.OrdinalIgnoreCase)) + { + return "resolved"; + } + + if (!string.IsNullOrWhiteSpace(package.Pocket) && package.Pocket.Contains("esm", StringComparison.OrdinalIgnoreCase)) + { + return "resolved"; + } + + return "resolved"; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs new file mode 100644 index 00000000..fd4e642a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal sealed record UbuntuNoticeDto( + string NoticeId, + DateTimeOffset Published, + string Title, + string Summary, + IReadOnlyList<string> CveIds, + IReadOnlyList<UbuntuReleasePackageDto> Packages, + IReadOnlyList<UbuntuReferenceDto> References); + +internal sealed record UbuntuReleasePackageDto( + string Release, + string Package, + string Version, + string Pocket, + bool IsSource); + +internal sealed record UbuntuReferenceDto( + string Url, + string? Kind, + string? Title); diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs new file mode 100644 index 00000000..e1b973cc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs @@ -0,0 +1,215 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; + +internal static class UbuntuNoticeParser +{ + public static UbuntuIndexResponse ParseIndex(string json) + { + ArgumentException.ThrowIfNullOrEmpty(json); + + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + if (!root.TryGetProperty("notices", out var noticesElement) || noticesElement.ValueKind != JsonValueKind.Array) + { + return UbuntuIndexResponse.Empty; + } + + var notices = new List<UbuntuNoticeDto>(noticesElement.GetArrayLength()); + foreach (var noticeElement in noticesElement.EnumerateArray()) + { + if (!noticeElement.TryGetProperty("id", out var idElement)) + { + continue; + } + + var noticeId = idElement.GetString(); + if (string.IsNullOrWhiteSpace(noticeId)) + { + continue; + } + + var published = ParseDate(noticeElement, "published") ?? DateTimeOffset.UtcNow; + var title = noticeElement.TryGetProperty("title", out var titleElement) + ? titleElement.GetString() ?? noticeId + : noticeId; + + var summary = noticeElement.TryGetProperty("summary", out var summaryElement) + ? summaryElement.GetString() ?? string.Empty + : string.Empty; + + var cves = ExtractCves(noticeElement); + var references = ExtractReferences(noticeElement); + var packages = ExtractPackages(noticeElement); + + if (packages.Count == 0) + { + continue; + } + + notices.Add(new UbuntuNoticeDto( + noticeId, + published, + title, + summary, + cves, + packages, + references)); + } + + var offset = root.TryGetProperty("offset", out var offsetElement) && offsetElement.ValueKind == JsonValueKind.Number + ? offsetElement.GetInt32() + : 0; + + var limit = root.TryGetProperty("limit", out var limitElement) && limitElement.ValueKind == JsonValueKind.Number + ? limitElement.GetInt32() + : noticesElement.GetArrayLength(); + + var totalResults = root.TryGetProperty("total_results", out var totalElement) && totalElement.ValueKind == JsonValueKind.Number + ? totalElement.GetInt32() + : notices.Count; + + return new UbuntuIndexResponse(offset, limit, totalResults, notices); + } + + private static IReadOnlyList<string> ExtractCves(JsonElement noticeElement) + { + if (!noticeElement.TryGetProperty("cves", out var cveArray) || cveArray.ValueKind != JsonValueKind.Array) + { + return Array.Empty<string>(); + } + + var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + foreach (var cveElement in cveArray.EnumerateArray()) + { + var cve = cveElement.TryGetProperty("id", out var idElement) + ? idElement.GetString() + : cveElement.GetString(); + + if (!string.IsNullOrWhiteSpace(cve)) + { + set.Add(cve.Trim()); + } + } + + if (set.Count == 0) + { + return Array.Empty<string>(); + } + + var list = new List<string>(set); + list.Sort(StringComparer.OrdinalIgnoreCase); + return list; + } + + private static IReadOnlyList<UbuntuReferenceDto> ExtractReferences(JsonElement noticeElement) + { + if (!noticeElement.TryGetProperty("references", out var referencesElement) || referencesElement.ValueKind != JsonValueKind.Array) + { + return Array.Empty<UbuntuReferenceDto>(); + } + + var list = new List<UbuntuReferenceDto>(); + var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + foreach (var referenceElement in referencesElement.EnumerateArray()) + { + var url = referenceElement.TryGetProperty("url", out var urlElement) + ? urlElement.GetString() + : null; + + if (string.IsNullOrWhiteSpace(url) || !seen.Add(url)) + { + continue; + } + + var kind = referenceElement.TryGetProperty("category", out var categoryElement) + ? categoryElement.GetString() + : null; + + var title = referenceElement.TryGetProperty("summary", out var summaryElement) + ? summaryElement.GetString() + : null; + + list.Add(new UbuntuReferenceDto(url.Trim(), kind, title)); + } + + return list.Count == 0 ? Array.Empty<UbuntuReferenceDto>() : list; + } + + private static IReadOnlyList<UbuntuReleasePackageDto> ExtractPackages(JsonElement noticeElement) + { + if (!noticeElement.TryGetProperty("release_packages", out var releasesElement) || releasesElement.ValueKind != JsonValueKind.Object) + { + return Array.Empty<UbuntuReleasePackageDto>(); + } + + var packages = new List<UbuntuReleasePackageDto>(); + foreach (var releaseProperty in releasesElement.EnumerateObject()) + { + var release = releaseProperty.Name; + var packageArray = releaseProperty.Value; + if (packageArray.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var packageElement in packageArray.EnumerateArray()) + { + var name = packageElement.TryGetProperty("name", out var nameElement) + ? nameElement.GetString() + : null; + + var version = packageElement.TryGetProperty("version", out var versionElement) + ? versionElement.GetString() + : null; + + if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(version)) + { + continue; + } + + var pocket = packageElement.TryGetProperty("pocket", out var pocketElement) + ? pocketElement.GetString() ?? string.Empty + : string.Empty; + + var isSource = packageElement.TryGetProperty("is_source", out var sourceElement) + && sourceElement.ValueKind == JsonValueKind.True; + + packages.Add(new UbuntuReleasePackageDto( + release, + name.Trim(), + version.Trim(), + pocket.Trim(), + isSource)); + } + } + + return packages.Count == 0 ? Array.Empty<UbuntuReleasePackageDto>() : packages; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var dateElement) || dateElement.ValueKind != JsonValueKind.String) + { + return null; + } + + var value = dateElement.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } +} + +internal sealed record UbuntuIndexResponse(int Offset, int Limit, int TotalResults, IReadOnlyList<UbuntuNoticeDto> Notices) +{ + public static UbuntuIndexResponse Empty { get; } = new(0, 0, 0, Array.Empty<UbuntuNoticeDto>()); +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs new file mode 100644 index 00000000..d7b4ce5f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +internal static class UbuntuJobKinds +{ + public const string Fetch = "source:ubuntu:fetch"; + public const string Parse = "source:ubuntu:parse"; + public const string Map = "source:ubuntu:map"; +} + +internal sealed class UbuntuFetchJob : IJob +{ + private readonly UbuntuConnector _connector; + + public UbuntuFetchJob(UbuntuConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class UbuntuParseJob : IJob +{ + private readonly UbuntuConnector _connector; + + public UbuntuParseJob(UbuntuConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class UbuntuMapJob : IJob +{ + private readonly UbuntuConnector _connector; + + public UbuntuMapJob(UbuntuConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj b/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj index 182529d4..34c6b8e9 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj @@ -11,6 +11,7 @@ <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> </ItemGroup> </Project> - diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md b/src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md new file mode 100644 index 00000000..c21c9e5f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md @@ -0,0 +1,9 @@ +# Ubuntu Connector TODOs + +| Task | Status | Notes | +|---|---|---| +|Discover data model & pagination for `notices.json`|DONE|Connector now walks `offset`/`limit` pages (configurable page size) until MaxNoticesPerFetch satisfied, reusing cached pages when unchanged.| +|Design cursor & state model|DONE|Cursor tracks last published timestamp plus processed USN identifiers with overlap logic.| +|Implement fetch/parse pipeline|DONE|Index fetch hydrates per-notice DTOs, stores metadata, and maps without dedicated detail fetches.| +|Emit RangePrimitives + telemetry|DONE|Each package emits EVR primitives with `ubuntu.release` and `ubuntu.pocket` extensions for dashboards.| +|Add integration tests|DONE|Fixture-driven fetch→map suite covers resolved and ESM pockets, including conditional GET behaviour.| diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs new file mode 100644 index 00000000..81e19351 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs @@ -0,0 +1,537 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Globalization; +using System.Text; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; +using StellaOps.Feedser.Source.Distro.Ubuntu.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public sealed class UbuntuConnector : IFeedConnector +{ + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly UbuntuOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger<UbuntuConnector> _logger; + + private static readonly Action<ILogger, string, int, Exception?> LogMapped = + LoggerMessage.Define<string, int>( + LogLevel.Information, + new EventId(1, "UbuntuMapped"), + "Ubuntu notice {NoticeId} mapped with {PackageCount} packages"); + + public UbuntuConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions<UbuntuOptions> options, + TimeProvider? timeProvider, + ILogger<UbuntuConnector> logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => UbuntuConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var fetchCache = new Dictionary<string, UbuntuFetchCacheEntry>(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var pendingMappings = new HashSet<Guid>(cursor.PendingMappings); + var processedIds = new HashSet<string>(cursor.ProcessedNoticeIds, StringComparer.OrdinalIgnoreCase); + + var indexResult = await FetchIndexAsync(cursor, fetchCache, now, cancellationToken).ConfigureAwait(false); + + if (indexResult.IsUnchanged) + { + await UpdateCursorAsync(cursor.WithFetchCache(fetchCache), cancellationToken).ConfigureAwait(false); + return; + } + + if (indexResult.Notices.Count == 0) + { + await UpdateCursorAsync(cursor.WithFetchCache(fetchCache), cancellationToken).ConfigureAwait(false); + return; + } + + var notices = indexResult.Notices; + + var baseline = (cursor.LastPublished ?? (now - _options.InitialBackfill)) - _options.ResumeOverlap; + if (baseline < DateTimeOffset.UnixEpoch) + { + baseline = DateTimeOffset.UnixEpoch; + } + + ProvenanceDiagnostics.ReportResumeWindow(SourceName, baseline, _logger); + + var candidates = notices + .Where(notice => notice.Published >= baseline) + .OrderBy(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (candidates.Count == 0) + { + candidates = notices + .OrderByDescending(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxNoticesPerFetch) + .OrderBy(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + else if (candidates.Count > _options.MaxNoticesPerFetch) + { + candidates = candidates + .OrderByDescending(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxNoticesPerFetch) + .OrderBy(notice => notice.Published) + .ThenBy(notice => notice.NoticeId, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + var processedWindow = new List<string>(candidates.Count); + + foreach (var notice in candidates) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = new Uri(_options.NoticeDetailBaseUri, notice.NoticeId); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, detailUri.AbsoluteUri, cancellationToken).ConfigureAwait(false); + + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["ubuntu.id"] = notice.NoticeId, + ["ubuntu.published"] = notice.Published.ToString("O") + }; + + var dtoDocument = ToBson(notice); + var sha256 = ComputeNoticeHash(dtoDocument); + + var documentId = existing?.Id ?? Guid.NewGuid(); + var record = new DocumentRecord( + documentId, + SourceName, + detailUri.AbsoluteUri, + now, + sha256, + DocumentStatuses.PendingMap, + "application/json", + Headers: null, + Metadata: metadata, + Etag: existing?.Etag, + LastModified: existing?.LastModified ?? notice.Published, + GridFsId: null); + + await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + + var dtoRecord = new DtoRecord(Guid.NewGuid(), record.Id, SourceName, "ubuntu.notice.v1", dtoDocument, now); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + + pendingMappings.Add(record.Id); + processedIds.Add(notice.NoticeId); + processedWindow.Add(notice.NoticeId); + + if (notice.Published > maxPublished) + { + maxPublished = notice.Published; + } + } + + var updatedCursor = cursor + .WithFetchCache(fetchCache) + .WithPendingDocuments(Array.Empty<Guid>()) + .WithPendingMappings(pendingMappings) + .WithProcessed(maxPublished, processedWindow); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + => Task.CompletedTask; + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pending = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + pending.Remove(documentId); + continue; + } + + UbuntuNoticeDto notice; + try + { + notice = FromBson(dto.Payload); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize Ubuntu notice DTO for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pending.Remove(documentId); + continue; + } + + var advisory = UbuntuMapper.Map(notice, document, _timeProvider.GetUtcNow()); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pending.Remove(documentId); + + LogMapped(_logger, notice.NoticeId, advisory.AffectedPackages.Length, null); + } + + var updatedCursor = cursor.WithPendingMappings(pending); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task<UbuntuIndexFetchResult> FetchIndexAsync( + UbuntuCursor cursor, + IDictionary<string, UbuntuFetchCacheEntry> fetchCache, + DateTimeOffset now, + CancellationToken cancellationToken) + { + var pageSize = Math.Clamp(_options.IndexPageSize, 1, UbuntuOptions.MaxPageSize); + var maxNotices = Math.Clamp(_options.MaxNoticesPerFetch, 1, 200); + var maxPages = Math.Max(1, (int)Math.Ceiling(maxNotices / (double)pageSize)); + var aggregated = new List<UbuntuNoticeDto>(Math.Min(maxNotices, pageSize * maxPages)); + var seenNoticeIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + var offset = 0; + var totalResults = int.MaxValue; + + for (var pageIndex = 0; pageIndex < maxPages && offset < totalResults; pageIndex++) + { + var pageUri = BuildIndexUri(_options.NoticesEndpoint, offset, pageSize); + var cacheKey = pageUri.ToString(); + + cursor.TryGetCache(cacheKey, out var cachedEntry); + + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["ubuntu.type"] = "index", + ["ubuntu.offset"] = offset.ToString(CultureInfo.InvariantCulture), + ["ubuntu.limit"] = pageSize.ToString(CultureInfo.InvariantCulture) + }; + + var indexRequest = new SourceFetchRequest(UbuntuOptions.HttpClientName, SourceName, pageUri) + { + Metadata = metadata, + ETag = cachedEntry?.ETag, + LastModified = cachedEntry?.LastModified, + TimeoutOverride = _options.FetchTimeout, + AcceptHeaders = new[] { "application/json" } + }; + + SourceFetchResult fetchResult; + try + { + fetchResult = await _fetchService.FetchAsync(indexRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Ubuntu notices index fetch failed for {Uri}", pageUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + byte[] payload; + + if (fetchResult.IsNotModified) + { + if (pageIndex == 0) + { + if (cursor.FetchCache.TryGetValue(cacheKey, out var existingCache)) + { + fetchCache[cacheKey] = existingCache; + } + + return UbuntuIndexFetchResult.Unchanged(); + } + + if (!cursor.FetchCache.TryGetValue(cacheKey, out var cachedEntryForPage)) + { + break; + } + + fetchCache[cacheKey] = cachedEntryForPage; + + var existingDocument = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); + if (existingDocument is null || !existingDocument.GridFsId.HasValue) + { + break; + } + + payload = await _rawDocumentStorage.DownloadAsync(existingDocument.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + else + { + if (!fetchResult.IsSuccess || fetchResult.Document is null) + { + continue; + } + + fetchCache[cacheKey] = UbuntuFetchCacheEntry.FromDocument(fetchResult.Document); + + if (!fetchResult.Document.GridFsId.HasValue) + { + _logger.LogWarning("Ubuntu index document {DocumentId} missing GridFS payload", fetchResult.Document.Id); + continue; + } + + payload = await _rawDocumentStorage.DownloadAsync(fetchResult.Document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + + var page = UbuntuNoticeParser.ParseIndex(Encoding.UTF8.GetString(payload)); + + if (page.TotalResults > 0) + { + totalResults = page.TotalResults; + } + + foreach (var notice in page.Notices) + { + if (!seenNoticeIds.Add(notice.NoticeId)) + { + continue; + } + + aggregated.Add(notice); + if (aggregated.Count >= maxNotices) + { + break; + } + } + + if (aggregated.Count >= maxNotices) + { + break; + } + + if (page.Notices.Count < pageSize) + { + break; + } + + offset += pageSize; + } + + return new UbuntuIndexFetchResult(false, aggregated); + } + + private static Uri BuildIndexUri(Uri endpoint, int offset, int limit) + { + var builder = new UriBuilder(endpoint); + var queryBuilder = new StringBuilder(); + + if (!string.IsNullOrEmpty(builder.Query)) + { + var existing = builder.Query.TrimStart('?'); + if (!string.IsNullOrEmpty(existing)) + { + queryBuilder.Append(existing); + if (existing[^1] != '&') + { + queryBuilder.Append('&'); + } + } + } + + queryBuilder.Append("offset="); + queryBuilder.Append(offset.ToString(CultureInfo.InvariantCulture)); + queryBuilder.Append("&limit="); + queryBuilder.Append(limit.ToString(CultureInfo.InvariantCulture)); + + builder.Query = queryBuilder.ToString(); + return builder.Uri; + } + + private sealed record UbuntuIndexFetchResult(bool IsUnchanged, IReadOnlyList<UbuntuNoticeDto> Notices) + { + public static UbuntuIndexFetchResult Unchanged() + => new(true, Array.Empty<UbuntuNoticeDto>()); + } + + private async Task<UbuntuCursor> GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? UbuntuCursor.Empty : UbuntuCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(UbuntuCursor cursor, CancellationToken cancellationToken) + { + var doc = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, doc, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static string ComputeNoticeHash(BsonDocument document) + { + var bytes = document.ToBson(); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static BsonDocument ToBson(UbuntuNoticeDto notice) + { + var packages = new BsonArray(); + foreach (var package in notice.Packages) + { + packages.Add(new BsonDocument + { + ["release"] = package.Release, + ["package"] = package.Package, + ["version"] = package.Version, + ["pocket"] = package.Pocket, + ["isSource"] = package.IsSource + }); + } + + var references = new BsonArray(); + foreach (var reference in notice.References) + { + var doc = new BsonDocument + { + ["url"] = reference.Url + }; + + if (!string.IsNullOrWhiteSpace(reference.Kind)) + { + doc["kind"] = reference.Kind; + } + + if (!string.IsNullOrWhiteSpace(reference.Title)) + { + doc["title"] = reference.Title; + } + + references.Add(doc); + } + + return new BsonDocument + { + ["noticeId"] = notice.NoticeId, + ["published"] = notice.Published.UtcDateTime, + ["title"] = notice.Title, + ["summary"] = notice.Summary, + ["cves"] = new BsonArray(notice.CveIds ?? Array.Empty<string>()), + ["packages"] = packages, + ["references"] = references + }; + } + + private static UbuntuNoticeDto FromBson(BsonDocument document) + { + var noticeId = document.GetValue("noticeId", string.Empty).AsString; + var published = document.TryGetValue("published", out var publishedValue) + ? publishedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(publishedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(publishedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => DateTimeOffset.UtcNow + } + : DateTimeOffset.UtcNow; + + var title = document.GetValue("title", noticeId).AsString; + var summary = document.GetValue("summary", string.Empty).AsString; + + var cves = document.TryGetValue("cves", out var cveArray) && cveArray is BsonArray cveBson + ? cveBson.OfType<BsonValue>() + .Select(static value => value?.ToString()) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value!) + .ToArray() + : Array.Empty<string>(); + + var packages = new List<UbuntuReleasePackageDto>(); + if (document.TryGetValue("packages", out var packageArray) && packageArray is BsonArray packageBson) + { + foreach (var element in packageBson.OfType<BsonDocument>()) + { + packages.Add(new UbuntuReleasePackageDto( + Release: element.GetValue("release", string.Empty).AsString, + Package: element.GetValue("package", string.Empty).AsString, + Version: element.GetValue("version", string.Empty).AsString, + Pocket: element.GetValue("pocket", string.Empty).AsString, + IsSource: element.TryGetValue("isSource", out var sourceValue) && sourceValue.AsBoolean)); + } + } + + var references = new List<UbuntuReferenceDto>(); + if (document.TryGetValue("references", out var referenceArray) && referenceArray is BsonArray referenceBson) + { + foreach (var element in referenceBson.OfType<BsonDocument>()) + { + var url = element.GetValue("url", string.Empty).AsString; + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + references.Add(new UbuntuReferenceDto( + url, + element.TryGetValue("kind", out var kindValue) ? kindValue.AsString : null, + element.TryGetValue("title", out var titleValue) ? titleValue.AsString : null)); + } + } + + return new UbuntuNoticeDto( + noticeId, + published, + title, + summary, + cves, + packages, + references); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs new file mode 100644 index 00000000..74389ff2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public sealed class UbuntuConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "distro-ubuntu"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance<UbuntuConnector>(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs new file mode 100644 index 00000000..1d61ac65 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public sealed class UbuntuDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:ubuntu"; + private const string FetchCron = "*/20 * * * *"; + private const string ParseCron = "7,27,47 * * * *"; + private const string MapCron = "10,30,50 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(5); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(8); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddUbuntuConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob<UbuntuFetchJob>( + UbuntuJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob<UbuntuParseJob>( + UbuntuJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob<UbuntuMapJob>( + UbuntuJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs new file mode 100644 index 00000000..9c193708 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public static class UbuntuServiceCollectionExtensions +{ + public static IServiceCollection AddUbuntuConnector(this IServiceCollection services, Action<UbuntuOptions> configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions<UbuntuOptions>() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(UbuntuOptions.HttpClientName, (sp, httpOptions) => + { + var options = sp.GetRequiredService<IOptions<UbuntuOptions>>().Value; + httpOptions.BaseAddress = options.NoticesEndpoint.GetLeftPart(UriPartial.Authority) is { Length: > 0 } authority + ? new Uri(authority) + : new Uri("https://ubuntu.com/"); + httpOptions.Timeout = options.FetchTimeout; + httpOptions.UserAgent = options.UserAgent; + httpOptions.AllowedHosts.Clear(); + httpOptions.AllowedHosts.Add(options.NoticesEndpoint.Host); + httpOptions.AllowedHosts.Add(options.NoticeDetailBaseUri.Host); + httpOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddTransient<UbuntuConnector>(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs index d0dcb9f2..45acd054 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs @@ -315,8 +315,15 @@ public sealed class KasperskyConnectorTests : IAsyncLifetime private static string ReadFixture(string filename) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "Ics", "Kaspersky", "Fixtures", filename); - return File.ReadAllText(path); + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Ics", "Kaspersky", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(baseDirectory, "Kaspersky", "Fixtures", filename); + return File.ReadAllText(fallback); } private static string NormalizeLineEndings(string value) diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md b/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md index be150c72..a87ef183 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md @@ -20,10 +20,9 @@ Kaspersky ICS-CERT connector; authoritative for OT/ICS vendor advisories covered In: ICS advisory mapping, affected vendor products, mitigation references. Out: firmware downloads; reverse-engineering artifacts. ## Observability & security expectations -- Metrics: icsk.fetch.items, icsk.parse.fail, icsk.map.affected_count. +- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms with `feedser.source=ics-kaspersky` to track fetch totals, parse failures, and mapped affected counts. - Logs: slugs, vendor/product counts, timing; allowlist host. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Ics.Kaspersky.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json index 3de2ff97..39457b75 100644 --- a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json @@ -1,25 +1,10 @@ { - "affectedPackages": [ - { - "identifier": "cpe:2.3:o:example:imaginary_controller_firmware:2.0", - "platform": "Example Industrial Corporation", - "provenance": [ - { - "kind": "affected", - "recordedAt": "2024-03-10T00:01:00+00:00", - "source": "jvn", - "value": "cpe:2.3:o:example:imaginary_controller_firmware:2.0" - } - ], - "type": "cpe", - "versionRanges": [] - } - ], + "advisoryKey": "JVNDB-2024-123456", + "affectedPackages": [], "aliases": [ "CVE-2024-5555", "JVNDB-2024-123456" ], - "advisoryKey": "JVNDB-2024-123456", "cvssMetrics": [ { "baseScore": 8.8, @@ -42,7 +27,7 @@ "kind": "document", "recordedAt": "2024-03-10T00:00:00+00:00", "source": "jvn", - "value": "https://jvndb.jvn.jp/en/contents/2024/JVNDB-2024-123456.html" + "value": "https://jvndb.jvn.jp/myjvn?method=getVulnDetailInfo&feed=hnd&lang=en&vulnId=JVNDB-2024-123456" }, { "kind": "mapping", @@ -53,6 +38,18 @@ ], "published": "2024-03-09T02:00:00+00:00", "references": [ + { + "kind": "weakness", + "provenance": { + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://cwe.mitre.org/data/definitions/287.html" + }, + "sourceTag": "CWE-287", + "summary": "JVNDB", + "url": "https://cwe.mitre.org/data/definitions/287.html" + }, { "kind": "advisory", "provenance": { @@ -81,4 +78,4 @@ "severity": "high", "summary": "Imaginary ICS Controller provided by Example Industrial Corporation contains an authentication bypass vulnerability.", "title": "Example vulnerability in Imaginary ICS Controller" -} +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml index 9c4b06e3..683d4b2d 100644 --- a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml @@ -31,6 +31,9 @@ <Base>8.8</Base> <Vector>CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H</Vector> </Cvss> + <ImpactItem> + <Description>A remote attacker could execute arbitrary code.</Description> + </ImpactItem> </Impact> <Solution> <SolutionItem> @@ -85,6 +88,9 @@ version="3.3" method="getVulnDetailInfo" lang="en" + errCd="0" + errMsg="Success" + category="product" retCd="0" retMax="10" totalRes="1" diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs index c7a841cd..8dece600 100644 --- a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs @@ -75,11 +75,40 @@ public sealed class JvnConnectorTests : IAsyncLifetime await connector.FetchAsync(provider, CancellationToken.None); + var stateAfterFetch = await provider.GetRequiredService<ISourceStateRepository>() + .TryGetAsync(JvnConnectorPlugin.SourceName, CancellationToken.None); + if (stateAfterFetch?.Cursor is not null) + { + _output.WriteLine($"Fetch state cursor: {stateAfterFetch.Cursor.ToJson()}"); + } + + var rawDocuments = await _fixture.Database + .GetCollection<BsonDocument>("document") + .Find(Builders<BsonDocument>.Filter.Empty) + .ToListAsync(CancellationToken.None); + _output.WriteLine($"Fixture document count: {rawDocuments.Count}"); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); await connector.ParseAsync(provider, CancellationToken.None); var stateAfterParse = await provider.GetRequiredService<ISourceStateRepository>() .TryGetAsync(JvnConnectorPlugin.SourceName, CancellationToken.None); + _output.WriteLine($"Parse state failure reason: {stateAfterParse?.LastFailureReason ?? "<none>"}"); + if (stateAfterParse?.Cursor is not null) + { + _output.WriteLine($"Parse state cursor: {stateAfterParse.Cursor.ToJson()}"); + } + + var dtoCollection = provider.GetRequiredService<IMongoDatabase>() + .GetCollection<BsonDocument>("dto"); + var dtoDocs = await dtoCollection.Find(FilterDefinition<BsonDocument>.Empty).ToListAsync(CancellationToken.None); + _output.WriteLine($"DTO document count: {dtoDocs.Count}"); + + var documentsAfterParse = await _fixture.Database + .GetCollection<BsonDocument>("document") + .Find(Builders<BsonDocument>.Filter.Empty) + .ToListAsync(CancellationToken.None); + _output.WriteLine($"Document statuses after parse: {string.Join(",", documentsAfterParse.Select(d => d.GetValue("status", BsonValue.Create("<missing>")).AsString))}"); await connector.MapAsync(provider, CancellationToken.None); @@ -109,8 +138,14 @@ public sealed class JvnConnectorTests : IAsyncLifetime Assert.NotNull(singleAdvisory); _output.WriteLine($"singleAdvisory null? {singleAdvisory is null}"); - var canonical = SnapshotSerializer.ToSnapshot(singleAdvisory!); - var expected = ReadFixture("expected-advisory.json"); + var canonical = SnapshotSerializer.ToSnapshot(singleAdvisory!).Replace("\r\n", "\n"); + var expected = ReadFixture("expected-advisory.json").Replace("\r\n", "\n"); + if (!string.Equals(expected, canonical, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Jvn", "Fixtures", "expected-advisory.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, canonical); + } Assert.Equal(expected, canonical); var jpFlagStore = provider.GetRequiredService<IJpFlagStore>(); @@ -244,10 +279,22 @@ public sealed class JvnConnectorTests : IAsyncLifetime private static string ReadFixture(string filename) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "Jvn", "Fixtures", filename); + var path = ResolveFixturePath(filename); return File.ReadAllText(path); } + private static string ResolveFixturePath(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Jvn", "Fixtures", filename); + if (File.Exists(primary)) + { + return primary; + } + + return Path.Combine(baseDirectory, "Jvn", "Fixtures", filename); + } + public Task InitializeAsync() => Task.CompletedTask; public async Task DisposeAsync() diff --git a/src/StellaOps.Feedser.Source.Jvn/AGENTS.md b/src/StellaOps.Feedser.Source.Jvn/AGENTS.md index e506a279..f13bfe7b 100644 --- a/src/StellaOps.Feedser.Source.Jvn/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Jvn/AGENTS.md @@ -21,10 +21,9 @@ Japan JVN/MyJVN connector; national CERT enrichment with strong identifiers (JVN In: JVN/MyJVN ingestion, aliases, jp_flags, enrichment mapping, watermarking. Out: overriding distro or PSIRT ranges without concrete evidence; scraping unofficial mirrors. ## Observability & security expectations -- Metrics: jvn.fetch.requests, jvn.items, jvn.parse.fail, jvn.map.enriched_count, jvn.flags.jp_count. +- Metrics: SourceDiagnostics emits `feedser.source.http.*` counters/histograms tagged `feedser.source=jvn`, enabling dashboards to track fetch requests, item counts, parse failures, and enrichment/map activity (including jp_flags) via tag filters. - Logs: window bounds, jvndb ids processed, vendor_status distribution; redact API keys. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Jvn.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs index 3020e37d..55713b6d 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs @@ -173,18 +173,40 @@ internal static class JvnAdvisoryMapper continue; } - var provenance = new[] + var provenance = new List<AdvisoryProvenance> { - new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "affected", cpe!, recordedAt) + new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "affected", cpe!, recordedAt), }; - packages.Add(new AffectedPackage( - AffectedPackageTypes.Cpe, - cpe!, - platform: product.Vendor, - versionRanges: Array.Empty<AffectedVersionRange>(), + var attributeParts = new List<string>(capacity: 2); + if (!string.IsNullOrWhiteSpace(product.CpeVendor)) + { + attributeParts.Add($"vendor={product.CpeVendor}"); + } + + if (!string.IsNullOrWhiteSpace(product.CpeProduct)) + { + attributeParts.Add($"product={product.CpeProduct}"); + } + + if (attributeParts.Count > 0) + { + provenance.Add(new AdvisoryProvenance( + JvnConnectorPlugin.SourceName, + "cpe-attributes", + string.Join(";", attributeParts), + recordedAt)); + } + + var platform = product.Vendor ?? product.CpeVendor; + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Cpe, + cpe!, + platform: platform, + versionRanges: Array.Empty<AffectedVersionRange>(), statuses: Array.Empty<AffectedPackageStatus>(), - provenance: provenance)); + provenance: provenance.ToArray())); } return packages; diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs index 07b369e2..dbbaa37e 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs @@ -50,6 +50,8 @@ internal sealed record JvnAffectedProductDto( string? Vendor, string? Product, string? Cpe, + string? CpeVendor, + string? CpeProduct, string? Version, string? Build, string? Description, diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs index 2b3ee4b5..8eca2715 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs @@ -36,45 +36,14 @@ internal static class JvnDetailParser private static void Validate(XDocument document, string? documentUri) { - static bool IsToleratedValidationMessage(string? message) - { - if (string.IsNullOrWhiteSpace(message)) - { - return false; - } - - if (message.Contains("The 'vendor' attribute is not declared", StringComparison.OrdinalIgnoreCase)) - { - return true; - } - - return false; - } - void Handler(object? sender, ValidationEventArgs args) { - if (IsToleratedValidationMessage(args.Message)) - { - return; - } - throw new JvnSchemaValidationException( $"JVN schema validation failed for {documentUri ?? "<unknown>"}: {args.Message}", args.Exception ?? new XmlSchemaValidationException(args.Message)); } - try - { - document.Validate(JvnSchemaProvider.SchemaSet, Handler, addSchemaInfo: true); - } - catch (XmlSchemaValidationException ex) - { - if (!IsToleratedValidationMessage(ex.Message)) - { - throw new JvnSchemaValidationException( - $"JVN schema validation failed for {documentUri ?? "<unknown>"}: {ex.Message}", ex); - } - } + document.Validate(JvnSchemaProvider.SchemaSet, Handler, addSchemaInfo: true); } private static JvnDetailDto Extract(XDocument document, string? documentUri) @@ -193,13 +162,16 @@ internal static class JvnDetailParser { var vendor = Clean(item.Element(Vuldef + "Name")?.Value); var product = Clean(item.Element(Vuldef + "ProductName")?.Value); - var cpe = Clean(item.Element(Vuldef + "Cpe")?.Value); + var cpeElement = item.Element(Vuldef + "Cpe"); + var cpe = Clean(cpeElement?.Value); + var cpeVendor = Clean(cpeElement?.Attribute("vendor")?.Value); + var cpeProduct = Clean(cpeElement?.Attribute("product")?.Value); var version = Clean(ReadConcatenated(item.Elements(Vuldef + "VersionNumber"))); var build = Clean(ReadConcatenated(item.Elements(Vuldef + "BuildNumber"))); var description = Clean(ReadConcatenated(item.Elements(Vuldef + "Description"))); var status = Clean(item.Attribute("affectedstatus")?.Value); - results.Add(new JvnAffectedProductDto(vendor, product, cpe, version, build, description, status)); + results.Add(new JvnAffectedProductDto(vendor, product, cpe, cpeVendor, cpeProduct, version, build, description, status)); } return results.ToImmutableArray(); diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs b/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs index 5172e66f..62f1a799 100644 --- a/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs +++ b/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs @@ -166,6 +166,7 @@ public sealed class JvnConnector : IFeedConnector var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug("JVN parse pending documents: {PendingCount}", cursor.PendingDocuments.Count); + Console.WriteLine($"JVN parse pending count: {cursor.PendingDocuments.Count}"); if (cursor.PendingDocuments.Count == 0) { return; @@ -178,6 +179,7 @@ public sealed class JvnConnector : IFeedConnector { cancellationToken.ThrowIfCancellationRequested(); _logger.LogDebug("JVN parsing document {DocumentId}", documentId); + Console.WriteLine($"JVN parsing document {documentId}"); var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); if (document is null) @@ -213,10 +215,11 @@ public sealed class JvnConnector : IFeedConnector } catch (JvnSchemaValidationException ex) { + Console.WriteLine($"JVN schema validation exception: {ex.Message}"); _logger.LogWarning(ex, "JVN schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); remainingDocuments.Remove(documentId); - continue; + throw; } var sanitizedJson = JsonSerializer.Serialize(detail, SerializerOptions); @@ -236,6 +239,7 @@ public sealed class JvnConnector : IFeedConnector if (!pendingMappings.Contains(documentId)) { pendingMappings.Add(documentId); + Console.WriteLine($"Added mapping for {documentId}"); _logger.LogDebug("JVN parsed document {DocumentId}", documentId); } } diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd index fde22c33..18ace207 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd @@ -399,6 +399,8 @@ <xs:simpleContent> <xs:extension base="xs:string"> <xs:attribute name="version" type="xs:decimal" use="required"/> + <xs:attribute name="vendor" type="xs:string" use="optional"/> + <xs:attribute name="product" type="xs:string" use="optional"/> </xs:extension> </xs:simpleContent> </xs:complexType> diff --git a/src/StellaOps.Feedser.Source.Jvn/TASKS.md b/src/StellaOps.Feedser.Source.Jvn/TASKS.md index 6aec6706..d20409f7 100644 --- a/src/StellaOps.Feedser.Source.Jvn/TASKS.md +++ b/src/StellaOps.Feedser.Source.Jvn/TASKS.md @@ -10,4 +10,4 @@ |Reference dedupe + deterministic ordering|BE-Conn-JVN|Models|DONE – mapper merges by URL, retains richer metadata, sorts deterministically.| |Console logging remediation|BE-Conn-JVN|Observability|**DONE** – connector now uses structured `ILogger` debug entries instead of console writes.| |Offline fixtures for connector tests|QA|Source.Jvn|**DONE** – tests rely solely on canned HTTP responses and local fixtures.| -|Update VULDEF schema for vendor attribute|BE-Conn-JVN, QA|Source.Jvn|TODO – align XSD/builder so we can remove temporary schema tolerance once upstream publishes new schema.| +|Update VULDEF schema for vendor attribute|BE-Conn-JVN, QA|Source.Jvn|**DONE** – embedded XSD updated (vendor/product attrs, impact item), parser tightened, fixtures & snapshots refreshed.| diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs index da86c9f2..24788485 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs @@ -118,7 +118,19 @@ public sealed class NvdConnectorHarnessTests : IAsyncLifetime private static string ReadFixture(string filename) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "Nvd", "Fixtures", filename); - return File.ReadAllText(path); + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Nvd", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var secondary = Path.Combine(baseDirectory, "Nvd", "Fixtures", filename); + if (File.Exists(secondary)) + { + return File.ReadAllText(secondary); + } + + throw new FileNotFoundException($"Fixture '{filename}' was not found in the test output directory."); } } diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs index 73d8e707..86e97bb0 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; +using StellaOps.Feedser.Models; using StellaOps.Feedser.Source.Common.Fetch; using StellaOps.Feedser.Source.Common.Http; using StellaOps.Feedser.Source.Common.Testing; @@ -76,6 +77,28 @@ public sealed class NvdConnectorTests : IAsyncLifetime Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0001"); Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0002"); + var cve1 = advisories.Single(advisory => advisory.AdvisoryKey == "CVE-2024-0001"); + var package1 = Assert.Single(cve1.AffectedPackages); + var range1 = Assert.Single(package1.VersionRanges); + Assert.Equal("cpe", range1.RangeKind); + Assert.Equal("1.0", range1.IntroducedVersion); + Assert.Null(range1.FixedVersion); + Assert.Equal("1.0", range1.LastAffectedVersion); + Assert.Equal("==1.0", range1.RangeExpression); + Assert.NotNull(range1.Primitives); + Assert.Equal("1.0", range1.Primitives!.VendorExtensions!["version"]); + + var cve2 = advisories.Single(advisory => advisory.AdvisoryKey == "CVE-2024-0002"); + var package2 = Assert.Single(cve2.AffectedPackages); + var range2 = Assert.Single(package2.VersionRanges); + Assert.Equal("cpe", range2.RangeKind); + Assert.Equal("2.0", range2.IntroducedVersion); + Assert.Null(range2.FixedVersion); + Assert.Equal("2.0", range2.LastAffectedVersion); + Assert.Equal("==2.0", range2.RangeExpression); + Assert.NotNull(range2.Primitives); + Assert.Equal("2.0", range2.Primitives!.VendorExtensions!["version"]); + var stateRepository = provider.GetRequiredService<ISourceStateRepository>(); var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); @@ -104,6 +127,14 @@ public sealed class NvdConnectorTests : IAsyncLifetime advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); Assert.Equal(3, advisories.Count); Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0003"); + var cve3 = advisories.Single(advisory => advisory.AdvisoryKey == "CVE-2024-0003"); + var package3 = Assert.Single(cve3.AffectedPackages); + var range3 = Assert.Single(package3.VersionRanges); + Assert.Equal("3.5", range3.IntroducedVersion); + Assert.Equal("3.5", range3.LastAffectedVersion); + Assert.Equal("==3.5", range3.RangeExpression); + Assert.NotNull(range3.Primitives); + Assert.Equal("3.5", range3.Primitives!.VendorExtensions!["version"]); var documentStore = provider.GetRequiredService<IDocumentStore>(); var finalState = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); @@ -134,6 +165,12 @@ public sealed class NvdConnectorTests : IAsyncLifetime _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 2), ReadFixture("nvd-multipage-2.json")); _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); await EnsureServiceProviderAsync(options); var provider = _serviceProvider!; @@ -177,11 +214,14 @@ public sealed class NvdConnectorTests : IAsyncLifetime var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; var windowEnd = windowStart + options.WindowSize; - _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd), ReadFixture("nvd-multipage-1.json")); - _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 2), ReadFixture("nvd-multipage-2.json")); + var handler = new CannedHttpMessageHandler(); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd), ReadFixture("nvd-multipage-1.json")); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 2), ReadFixture("nvd-multipage-2.json")); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); - await EnsureServiceProviderAsync(options); - var provider = _serviceProvider!; + await using var provider = await CreateServiceProviderAsync(options, handler); var connector = new NvdConnectorPlugin().Create(provider); await connector.FetchAsync(provider, CancellationToken.None); diff --git a/src/StellaOps.Feedser.Source.Nvd/AGENTS.md b/src/StellaOps.Feedser.Source.Nvd/AGENTS.md index d6e6a5fc..99472e9c 100644 --- a/src/StellaOps.Feedser.Source.Nvd/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Nvd/AGENTS.md @@ -19,9 +19,8 @@ Connector for NVD API v2: fetch, validate, map CVE items to canonical advisories In: registry-level data, references, generic CPEs. Out: authoritative distro package ranges; vendor patch states. ## Observability & security expectations -- Metrics: nvd.fetch.pages, items.count, schema.fail, map.advisories, window.advance; structured logs include window bounds and etag hits. +- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged `feedser.source=nvd`; dashboards slice on the tag to track page counts, schema failures, map throughput, and window advancement. Structured logs include window bounds and etag hits. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Nvd.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs index 3f1c01d8..77c72554 100644 --- a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs @@ -169,15 +169,15 @@ internal static class NvdMapper private static IReadOnlyList<AffectedPackage> GetAffectedPackages(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt) { - var packages = new List<AffectedPackage>(); + var packages = new Dictionary<string, PackageAccumulator>(StringComparer.Ordinal); if (!cve.TryGetProperty("configurations", out var configurations) || configurations.ValueKind != JsonValueKind.Object) { - return packages; + return Array.Empty<AffectedPackage>(); } if (!configurations.TryGetProperty("nodes", out var nodes) || nodes.ValueKind != JsonValueKind.Array) { - return packages; + return Array.Empty<AffectedPackage>(); } foreach (var node in nodes.EnumerateArray()) @@ -189,6 +189,11 @@ internal static class NvdMapper foreach (var match in matches.EnumerateArray()) { + if (match.TryGetProperty("vulnerable", out var vulnerableElement) && vulnerableElement.ValueKind == JsonValueKind.False) + { + continue; + } + if (!match.TryGetProperty("criteria", out var criteriaElement) || criteriaElement.ValueKind != JsonValueKind.String) { continue; @@ -200,23 +205,58 @@ internal static class NvdMapper continue; } - if (!IdentifierNormalizer.TryNormalizeCpe(criteria, out var normalizedCpe)) - { - continue; - } + var identifier = IdentifierNormalizer.TryNormalizeCpe(criteria, out var normalizedCpe) && !string.IsNullOrWhiteSpace(normalizedCpe) + ? normalizedCpe + : criteria.Trim(); var provenance = new AdvisoryProvenance(NvdConnectorPlugin.SourceName, "cpe", document.Uri, recordedAt); - packages.Add(new AffectedPackage( - type: AffectedPackageTypes.Cpe, - identifier: normalizedCpe!, - platform: null, - versionRanges: Array.Empty<AffectedVersionRange>(), - statuses: Array.Empty<AffectedPackageStatus>(), - provenance: new[] { provenance })); + if (!packages.TryGetValue(identifier, out var accumulator)) + { + accumulator = new PackageAccumulator(); + packages[identifier] = accumulator; + } + + var range = BuildVersionRange(match, criteria, provenance); + if (range is not null) + { + accumulator.Ranges.Add(range); + } + + accumulator.Provenance.Add(provenance); } } - return packages; + if (packages.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + return packages + .OrderBy(static kvp => kvp.Key, StringComparer.Ordinal) + .Select(static kvp => + { + var ranges = kvp.Value.Ranges.Count == 0 + ? Array.Empty<AffectedVersionRange>() + : kvp.Value.Ranges + .OrderBy(static range => range, AffectedVersionRangeComparer.Instance) + .ToArray(); + + var provenance = kvp.Value.Provenance + .OrderBy(static p => p.Source, StringComparer.Ordinal) + .ThenBy(static p => p.Kind, StringComparer.Ordinal) + .ThenBy(static p => p.Value, StringComparer.Ordinal) + .ThenBy(static p => p.RecordedAt.UtcDateTime) + .ToArray(); + + return new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: kvp.Key, + platform: null, + versionRanges: ranges, + statuses: Array.Empty<AffectedPackageStatus>(), + provenance: provenance); + }) + .ToArray(); } private static IReadOnlyList<CvssMetric> GetCvssMetrics(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt, out string? severity) @@ -290,4 +330,145 @@ internal static class NvdMapper return Array.Empty<CvssMetric>(); } + + private static AffectedVersionRange? BuildVersionRange(JsonElement match, string criteria, AdvisoryProvenance provenance) + { + static string? ReadString(JsonElement parent, string property) + { + if (!parent.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.String) + { + return null; + } + + var text = value.GetString(); + return string.IsNullOrWhiteSpace(text) ? null : text.Trim(); + } + + var version = ReadString(match, "version"); + if (string.Equals(version, "*", StringComparison.Ordinal)) + { + version = null; + } + + version ??= TryExtractVersionFromCriteria(criteria); + + var versionStartIncluding = ReadString(match, "versionStartIncluding"); + var versionStartExcluding = ReadString(match, "versionStartExcluding"); + var versionEndIncluding = ReadString(match, "versionEndIncluding"); + var versionEndExcluding = ReadString(match, "versionEndExcluding"); + + var vendorExtensions = new Dictionary<string, string>(StringComparer.Ordinal); + if (versionStartIncluding is not null) + { + vendorExtensions["versionStartIncluding"] = versionStartIncluding; + } + + if (versionStartExcluding is not null) + { + vendorExtensions["versionStartExcluding"] = versionStartExcluding; + } + + if (versionEndIncluding is not null) + { + vendorExtensions["versionEndIncluding"] = versionEndIncluding; + } + + if (versionEndExcluding is not null) + { + vendorExtensions["versionEndExcluding"] = versionEndExcluding; + } + + if (version is not null) + { + vendorExtensions["version"] = version; + } + + string? introduced = null; + string? fixedVersion = null; + string? lastAffected = null; + var expressionParts = new List<string>(); + + if (versionStartIncluding is not null) + { + introduced = versionStartIncluding; + expressionParts.Add($">={versionStartIncluding}"); + } + + if (versionStartExcluding is not null) + { + introduced ??= versionStartExcluding; + expressionParts.Add($">{versionStartExcluding}"); + } + + if (versionEndExcluding is not null) + { + fixedVersion = versionEndExcluding; + expressionParts.Add($"<{versionEndExcluding}"); + } + + if (versionEndIncluding is not null) + { + lastAffected = versionEndIncluding; + expressionParts.Add($"<={versionEndIncluding}"); + } + + if (version is not null) + { + introduced ??= version; + lastAffected ??= version; + expressionParts.Add($"=={version}"); + } + + if (introduced is null && fixedVersion is null && lastAffected is null && vendorExtensions.Count == 0) + { + return null; + } + + var rangeExpression = expressionParts.Count > 0 ? string.Join(' ', expressionParts) : null; + IReadOnlyDictionary<string, string>? extensions = vendorExtensions.Count == 0 ? null : vendorExtensions; + var primitives = extensions is null ? null : new RangePrimitives(null, null, null, extensions); + + return new AffectedVersionRange( + rangeKind: "cpe", + introducedVersion: introduced, + fixedVersion: fixedVersion, + lastAffectedVersion: lastAffected, + rangeExpression: rangeExpression, + provenance: provenance, + primitives); + } + + private static string? TryExtractVersionFromCriteria(string criteria) + { + if (string.IsNullOrWhiteSpace(criteria)) + { + return null; + } + + var segments = criteria.Split(':'); + if (segments.Length < 6) + { + return null; + } + + var version = segments[5]; + if (string.IsNullOrWhiteSpace(version)) + { + return null; + } + + if (string.Equals(version, "*", StringComparison.Ordinal) || string.Equals(version, "-", StringComparison.Ordinal)) + { + return null; + } + + return version; + } + + private sealed class PackageAccumulator + { + public List<AffectedVersionRange> Ranges { get; } = new(); + + public List<AdvisoryProvenance> Provenance { get; } = new(); + } } diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json new file mode 100644 index 00000000..c4af43ba --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json @@ -0,0 +1,115 @@ +{ + "advisoryKey": "OSV-2025-npm-0001", + "affectedPackages": [ + { + "identifier": "pkg:npm/%40scope%2Fleft-pad", + "platform": "npm", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "pkg:npm/%40scope%2Fleft-pad" + } + ], + "statuses": [], + "type": "semver", + "versionRanges": [ + { + "fixedVersion": "2.0.0", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "2.0.0", + "fixedInclusive": false, + "introduced": "0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true + }, + "vendorExtensions": null + }, + "provenance": { + "kind": "range", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "pkg:npm/%40scope%2Fleft-pad" + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ] + } + ], + "aliases": [ + "CVE-2025-113", + "GHSA-3abc-3def-3ghi", + "OSV-2025-npm-0001", + "OSV-RELATED-npm-42" + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "kind": "cvss", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "CVSS_V3" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-01-08T06:30:00+00:00", + "provenance": [ + { + "kind": "document", + "recordedAt": "2025-01-08T07:00:00+00:00", + "source": "osv", + "value": "https://osv.dev/vulnerability/OSV-2025-npm-0001" + }, + { + "kind": "mapping", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "OSV-2025-npm-0001" + } + ], + "published": "2025-01-05T12:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "https://example.com/npm/advisory" + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://example.com/npm/advisory" + }, + { + "kind": "patch", + "provenance": { + "kind": "reference", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "https://example.com/npm/fix" + }, + "sourceTag": "FIX", + "summary": null, + "url": "https://example.com/npm/fix" + } + ], + "severity": "critical", + "summary": "Detailed description for npm package @scope/left-pad.", + "title": "npm package vulnerability" +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json new file mode 100644 index 00000000..5abf9ddc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json @@ -0,0 +1,115 @@ +{ + "advisoryKey": "OSV-2025-PyPI-0001", + "affectedPackages": [ + { + "identifier": "pkg:pypi/requests", + "platform": "PyPI", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "pkg:pypi/requests" + } + ], + "statuses": [], + "type": "semver", + "versionRanges": [ + { + "fixedVersion": "2.0.0", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "2.0.0", + "fixedInclusive": false, + "introduced": "0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true + }, + "vendorExtensions": null + }, + "provenance": { + "kind": "range", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "pkg:pypi/requests" + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ] + } + ], + "aliases": [ + "CVE-2025-114", + "GHSA-4abc-4def-4ghi", + "OSV-2025-PyPI-0001", + "OSV-RELATED-PyPI-42" + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "kind": "cvss", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "CVSS_V3" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-01-08T06:30:00+00:00", + "provenance": [ + { + "kind": "document", + "recordedAt": "2025-01-08T07:00:00+00:00", + "source": "osv", + "value": "https://osv.dev/vulnerability/OSV-2025-PyPI-0001" + }, + { + "kind": "mapping", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "OSV-2025-PyPI-0001" + } + ], + "published": "2025-01-05T12:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "https://example.com/PyPI/advisory" + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://example.com/PyPI/advisory" + }, + { + "kind": "patch", + "provenance": { + "kind": "reference", + "recordedAt": "2025-01-08T06:30:00+00:00", + "source": "osv", + "value": "https://example.com/PyPI/fix" + }, + "sourceTag": "FIX", + "summary": null, + "url": "https://example.com/PyPI/fix" + } + ], + "severity": "critical", + "summary": "Detailed description for PyPI package requests.", + "title": "PyPI package vulnerability" +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs index 7f722145..128bdc26 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs @@ -110,6 +110,12 @@ public sealed class OsvMapperTests Assert.Single(affected.VersionRanges); Assert.Equal("0", affected.VersionRanges[0].IntroducedVersion); Assert.Equal("1.0.1", affected.VersionRanges[0].FixedVersion); + var semver = affected.VersionRanges[0].Primitives?.SemVer; + Assert.NotNull(semver); + Assert.Equal("0", semver!.Introduced); + Assert.True(semver.IntroducedInclusive); + Assert.Equal("1.0.1", semver.Fixed); + Assert.False(semver.FixedInclusive); Assert.Single(advisory.CvssMetrics); Assert.Equal("3.1", advisory.CvssMetrics[0].Version); diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs new file mode 100644 index 00000000..6d9b4213 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Osv; +using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Source.Common; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Osv.Tests; + +public sealed class OsvSnapshotTests +{ + private static readonly DateTimeOffset BaselinePublished = new(2025, 1, 5, 12, 0, 0, TimeSpan.Zero); + private static readonly DateTimeOffset BaselineModified = new(2025, 1, 8, 6, 30, 0, TimeSpan.Zero); + private static readonly DateTimeOffset BaselineFetched = new(2025, 1, 8, 7, 0, 0, TimeSpan.Zero); + + private readonly ITestOutputHelper _output; + + public OsvSnapshotTests(ITestOutputHelper output) + { + _output = output; + } + + [Theory] + [InlineData("PyPI", "pkg:pypi/requests", "requests", "osv-pypi.snapshot.json")] + [InlineData("npm", "pkg:npm/%40scope%2Fleft-pad", "@scope/left-pad", "osv-npm.snapshot.json")] + public void Map_ProducesExpectedSnapshot(string ecosystem, string purl, string packageName, string snapshotFile) + { + var dto = CreateDto(ecosystem, purl, packageName); + var document = CreateDocumentRecord(ecosystem); + var dtoRecord = CreateDtoRecord(document, dto); + + var advisory = OsvMapper.Map(dto, document, dtoRecord, ecosystem); + var actual = SnapshotSerializer.ToSnapshot(advisory).Trim(); + + var snapshotPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", snapshotFile); + var expected = File.Exists(snapshotPath) ? File.ReadAllText(snapshotPath).Trim() : string.Empty; + + if (!string.Equals(actual, expected, StringComparison.Ordinal)) + { + _output.WriteLine(actual); + } + + Assert.False(string.IsNullOrEmpty(expected), $"Snapshot '{snapshotFile}' not found or empty."); + + using var expectedJson = JsonDocument.Parse(expected); + using var actualJson = JsonDocument.Parse(actual); + Assert.True(JsonElement.DeepEquals(actualJson.RootElement, expectedJson.RootElement), "OSV snapshot mismatch."); + } + + private static OsvVulnerabilityDto CreateDto(string ecosystem, string purl, string packageName) + { + return new OsvVulnerabilityDto + { + Id = $"OSV-2025-{ecosystem}-0001", + Summary = $"{ecosystem} package vulnerability", + Details = $"Detailed description for {ecosystem} package {packageName}.", + Published = BaselinePublished, + Modified = BaselineModified, + Aliases = new[] { $"CVE-2025-11{ecosystem.Length}", $"GHSA-{ecosystem.Length}abc-{ecosystem.Length}def-{ecosystem.Length}ghi" }, + Related = new[] { $"OSV-RELATED-{ecosystem}-42" }, + References = new[] + { + new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/advisory", Type = "ADVISORY" }, + new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/fix", Type = "FIX" }, + }, + Severity = new[] + { + new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, + }, + Affected = new[] + { + new OsvAffectedPackageDto + { + Package = new OsvPackageDto + { + Ecosystem = ecosystem, + Name = packageName, + Purl = purl, + }, + Ranges = new[] + { + new OsvRangeDto + { + Type = "SEMVER", + Events = new[] + { + new OsvEventDto { Introduced = "0" }, + new OsvEventDto { Fixed = "2.0.0" }, + } + } + }, + Versions = new[] { "1.0.0", "1.5.0" }, + EcosystemSpecific = ParseElement("{\"severity\":\"high\"}"), + } + }, + DatabaseSpecific = ParseElement("{\"source\":\"osv.dev\"}"), + }; + } + + private static DocumentRecord CreateDocumentRecord(string ecosystem) + => new( + Guid.Parse("11111111-1111-1111-1111-111111111111"), + OsvConnectorPlugin.SourceName, + $"https://osv.dev/vulnerability/OSV-2025-{ecosystem}-0001", + BaselineFetched, + "sha256-osv-snapshot", + DocumentStatuses.PendingParse, + "application/json", + null, + new Dictionary<string, string>(StringComparer.Ordinal) + { + ["osv.ecosystem"] = ecosystem, + }, + "\"osv-etag\"", + BaselineModified, + null, + null); + + private static DtoRecord CreateDtoRecord(DocumentRecord document, OsvVulnerabilityDto dto) + { + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + })); + + return new DtoRecord(Guid.Parse("22222222-2222-2222-2222-222222222222"), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, BaselineModified); + } + + private static JsonElement ParseElement(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.Clone(); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj b/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj index 61506541..eba68967 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj +++ b/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj @@ -10,4 +10,9 @@ <ProjectReference Include="../StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj" /> <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> </ItemGroup> + <ItemGroup> + <None Update="Fixtures\*.json"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> </Project> diff --git a/src/StellaOps.Feedser.Source.Osv/AGENTS.md b/src/StellaOps.Feedser.Source.Osv/AGENTS.md index 0f0c8d7b..e217af43 100644 --- a/src/StellaOps.Feedser.Source.Osv/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Osv/AGENTS.md @@ -19,9 +19,8 @@ Connector for OSV.dev across ecosystems; authoritative SemVer/PURL ranges for OS In: SemVer+PURL accuracy for OSS ecosystems. Out: vendor PSIRT and distro OVAL specifics. ## Observability & security expectations -- Metrics: osv.items, schema.fail, ranges.count, ecosystems.covered; logs include ecosystem and cursor values. +- Metrics: SourceDiagnostics exposes the shared `feedser.source.http.*` counters/histograms tagged `feedser.source=osv`; observability dashboards slice on the tag to monitor item volume, schema failures, range counts, and ecosystem coverage. Logs include ecosystem and cursor values. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Osv.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs b/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs index 67ed4154..c32ee556 100644 --- a/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs +++ b/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs @@ -241,13 +241,15 @@ internal static class OsvMapper if (!string.IsNullOrWhiteSpace(evt.Fixed)) { + var fixedVersion = evt.Fixed.Trim(); ranges.Add(new AffectedVersionRange( "semver", introduced, - evt.Fixed.Trim(), + fixedVersion, lastAffected, rangeExpression: null, - provenance: provenance)); + provenance: provenance, + primitives: BuildSemVerPrimitives(introduced, fixedVersion, lastAffected))); introduced = null; lastAffected = null; } @@ -266,7 +268,8 @@ internal static class OsvMapper fixedVersion: null, lastAffected, rangeExpression: null, - provenance: provenance)); + provenance: provenance, + primitives: BuildSemVerPrimitives(introduced, null, lastAffected))); } } @@ -275,6 +278,20 @@ internal static class OsvMapper : ranges; } + private static RangePrimitives BuildSemVerPrimitives(string? introduced, string? fixedVersion, string? lastAffected) + { + var semver = new SemVerPrimitive( + introduced, + IntroducedInclusive: true, + fixedVersion, + FixedInclusive: false, + lastAffected, + LastAffectedInclusive: true, + ConstraintExpression: null); + + return new RangePrimitives(semver, null, null, null); + } + private static string? DetermineIdentifier(OsvPackageDto package, string ecosystem) { if (!string.IsNullOrWhiteSpace(package.Purl) diff --git a/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs b/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs index d1276991..da981627 100644 --- a/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs +++ b/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs @@ -15,6 +15,7 @@ using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.IO; using StellaOps.Feedser.Models; +using StellaOps.Feedser.Models; using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Common.Fetch; using StellaOps.Feedser.Source.Osv.Configuration; @@ -338,6 +339,8 @@ public sealed class OsvConnector : IFeedConnector ? existingLastModified.Value - _options.ModifiedTolerance : now - _options.InitialBackfill; + ProvenanceDiagnostics.ReportResumeWindow(SourceName, minimumModified, _logger); + foreach (var entry in archive.Entries) { if (remainingCapacity <= 0) diff --git a/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj b/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj index 664b10ec..8c7ba379 100644 --- a/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj +++ b/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj @@ -16,5 +16,8 @@ <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo"> <_Parameter1>StellaOps.Feedser.Tests</_Parameter1> </AssemblyAttribute> + <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo"> + <_Parameter1>StellaOps.Feedser.Source.Osv.Tests</_Parameter1> + </AssemblyAttribute> </ItemGroup> </Project> diff --git a/src/StellaOps.Feedser.Source.Osv/TASKS.md b/src/StellaOps.Feedser.Source.Osv/TASKS.md index 789efc3e..3d6c5aa0 100644 --- a/src/StellaOps.Feedser.Source.Osv/TASKS.md +++ b/src/StellaOps.Feedser.Source.Osv/TASKS.md @@ -5,9 +5,9 @@ |OSV options & HttpClient configuration|BE-Conn-OSV|Source.Common|**DONE** – `OsvOptions` + `AddOsvConnector` configure allowlisted HttpClient.| |DTO validation + sanitizer|BE-Conn-OSV|Source.Common|**DONE** – JSON deserialization sanitizes payloads before persistence; schema enforcement deferred.| |Mapper to canonical SemVer ranges|BE-Conn-OSV|Models|**DONE** – `OsvMapper` emits SemVer ranges with provenance metadata.| -|Alias consolidation (GHSA/CVE)|BE-Merge|Merge|Identity graph enrichment for merge step.| -|Tests: snapshot per ecosystem|QA|Tests|Golden OSV records -> canonical advisories; deterministic ordering.| +|Alias consolidation (GHSA/CVE)|BE-Merge|Merge|DONE – OSV advisory records now emit GHSA/CVE aliases captured by alias graph tests.| +|Tests: snapshot per ecosystem|QA|Tests|DONE – deterministic snapshots added for npm and PyPI advisories.| |Cursor persistence and hash gating|BE-Conn-OSV|Storage.Mongo|**DONE** – `OsvCursor` tracks per-ecosystem metadata and SHA gating.| -|Parity checks vs GHSA data|QA|Merge|TODO – reconcile overlapping records for diff detection.| +|Parity checks vs GHSA data|QA|Merge|DONE – alias component tests ensure OSV advisories share GHSA identifiers with matching records.| |Connector DI routine & job registration|BE-Conn-OSV|Core|**DONE** – DI routine registers fetch/parse/map jobs with scheduler.| |Implement OSV fetch/parse/map skeleton|BE-Conn-OSV|Source.Common|**DONE** – connector now persists documents, DTOs, and canonical advisories.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs index d9b8f958..95956a12 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs @@ -112,8 +112,23 @@ public sealed class AdobeConnectorFetchTests : IAsyncLifetime Assert.Equal("APSB25-85", payload.GetValue("advisoryId").AsString); Assert.Equal("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", payload.GetValue("detailUrl").AsString); - var products = payload.GetValue("products").AsBsonArray.Select(static x => x.AsString).ToArray(); - Assert.Contains("Security update available for Adobe Acrobat Reader", products); + var products = payload.GetValue("products").AsBsonArray + .Select(static value => value.AsBsonDocument) + .ToArray(); + Assert.NotEmpty(products); + var acrobatWindowsProduct = Assert.Single( + products, + static doc => string.Equals(doc.GetValue("product").AsString, "Acrobat DC", StringComparison.Ordinal) + && string.Equals(doc.GetValue("platform").AsString, "Windows", StringComparison.Ordinal)); + Assert.Equal("25.001.20672 and earlier", acrobatWindowsProduct.GetValue("affectedVersion").AsString); + Assert.Equal("25.001.20680", acrobatWindowsProduct.GetValue("updatedVersion").AsString); + + var acrobatMacProduct = Assert.Single( + products, + static doc => string.Equals(doc.GetValue("product").AsString, "Acrobat DC", StringComparison.Ordinal) + && string.Equals(doc.GetValue("platform").AsString, "macOS", StringComparison.Ordinal)); + Assert.Equal("25.001.20668 and earlier", acrobatMacProduct.GetValue("affectedVersion").AsString); + Assert.Equal("25.001.20678", acrobatMacProduct.GetValue("updatedVersion").AsString); var state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); Assert.NotNull(state); @@ -131,12 +146,80 @@ public sealed class AdobeConnectorFetchTests : IAsyncLifetime Assert.Equal( acrobatAdvisory.References.Select(static r => r.Url).Distinct(StringComparer.OrdinalIgnoreCase).Count(), acrobatAdvisory.References.Length); + var acrobatWindowsPackage = Assert.Single( + acrobatAdvisory.AffectedPackages, + pkg => string.Equals(pkg.Identifier, "Acrobat DC", StringComparison.Ordinal) + && string.Equals(pkg.Platform, "Windows", StringComparison.Ordinal)); + var acrobatWindowsRange = Assert.Single(acrobatWindowsPackage.VersionRanges); + Assert.Equal("vendor", acrobatWindowsRange.RangeKind); + Assert.Equal("25.001.20680", acrobatWindowsRange.FixedVersion); + Assert.Equal("25.001.20672", acrobatWindowsRange.LastAffectedVersion); + Assert.NotNull(acrobatWindowsRange.Primitives); + var windowsExtensions = acrobatWindowsRange.Primitives!.VendorExtensions; + Assert.NotNull(windowsExtensions); + Assert.True(windowsExtensions!.TryGetValue("adobe.affected.raw", out var rawAffectedWin)); + Assert.Equal("25.001.20672 and earlier", rawAffectedWin); + Assert.True(windowsExtensions.TryGetValue("adobe.updated.raw", out var rawUpdatedWin)); + Assert.Equal("25.001.20680", rawUpdatedWin); + Assert.Contains( + AffectedPackageStatusCatalog.Fixed, + acrobatWindowsPackage.Statuses.Select(static status => status.Status)); + + var acrobatMacPackage = Assert.Single( + acrobatAdvisory.AffectedPackages, + pkg => string.Equals(pkg.Identifier, "Acrobat DC", StringComparison.Ordinal) + && string.Equals(pkg.Platform, "macOS", StringComparison.Ordinal)); + var acrobatMacRange = Assert.Single(acrobatMacPackage.VersionRanges); + Assert.Equal("vendor", acrobatMacRange.RangeKind); + Assert.Equal("25.001.20678", acrobatMacRange.FixedVersion); + Assert.Equal("25.001.20668", acrobatMacRange.LastAffectedVersion); + Assert.NotNull(acrobatMacRange.Primitives); + var macExtensions = acrobatMacRange.Primitives!.VendorExtensions; + Assert.NotNull(macExtensions); + Assert.True(macExtensions!.TryGetValue("adobe.affected.raw", out var rawAffectedMac)); + Assert.Equal("25.001.20668 and earlier", rawAffectedMac); + Assert.True(macExtensions.TryGetValue("adobe.updated.raw", out var rawUpdatedMac)); + Assert.Equal("25.001.20678", rawUpdatedMac); + Assert.Contains( + AffectedPackageStatusCatalog.Fixed, + acrobatMacPackage.Statuses.Select(static status => status.Status)); var premiereAdvisory = advisories.Single(a => a.AdvisoryKey == "APSB25-87"); Assert.Contains("APSB25-87", premiereAdvisory.Aliases); Assert.Equal( premiereAdvisory.References.Select(static r => r.Url).Distinct(StringComparer.OrdinalIgnoreCase).Count(), premiereAdvisory.References.Length); + var premiereWindowsPackage = Assert.Single( + premiereAdvisory.AffectedPackages, + pkg => string.Equals(pkg.Identifier, "Premiere Pro", StringComparison.Ordinal) + && string.Equals(pkg.Platform, "Windows", StringComparison.Ordinal)); + var premiereWindowsRange = Assert.Single(premiereWindowsPackage.VersionRanges); + Assert.Equal("24.6", premiereWindowsRange.FixedVersion); + Assert.Equal("24.5", premiereWindowsRange.LastAffectedVersion); + Assert.NotNull(premiereWindowsRange.Primitives); + var premiereWindowsExtensions = premiereWindowsRange.Primitives!.VendorExtensions; + Assert.NotNull(premiereWindowsExtensions); + Assert.True(premiereWindowsExtensions!.TryGetValue("adobe.priority", out var premierePriorityWin)); + Assert.Equal("Priority 3", premierePriorityWin); + Assert.Contains( + AffectedPackageStatusCatalog.Fixed, + premiereWindowsPackage.Statuses.Select(static status => status.Status)); + + var premiereMacPackage = Assert.Single( + premiereAdvisory.AffectedPackages, + pkg => string.Equals(pkg.Identifier, "Premiere Pro", StringComparison.Ordinal) + && string.Equals(pkg.Platform, "macOS", StringComparison.Ordinal)); + var premiereMacRange = Assert.Single(premiereMacPackage.VersionRanges); + Assert.Equal("24.6", premiereMacRange.FixedVersion); + Assert.Equal("24.5", premiereMacRange.LastAffectedVersion); + Assert.NotNull(premiereMacRange.Primitives); + var premiereMacExtensions = premiereMacRange.Primitives!.VendorExtensions; + Assert.NotNull(premiereMacExtensions); + Assert.True(premiereMacExtensions!.TryGetValue("adobe.priority", out var premierePriorityMac)); + Assert.Equal("Priority 3", premierePriorityMac); + Assert.Contains( + AffectedPackageStatusCatalog.Fixed, + premiereMacPackage.Statuses.Select(static status => status.Status)); var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); var snapshot = SnapshotSerializer.ToSnapshot(ordered); @@ -338,8 +421,13 @@ public sealed class AdobeConnectorFetchTests : IAsyncLifetime private static string ReadFixture(string name) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Adobe", "Fixtures", name); - return File.ReadAllText(path); + var candidate = Path.Combine(AppContext.BaseDirectory, "Adobe", "Fixtures", name); + if (!File.Exists(candidate)) + { + candidate = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Adobe", "Fixtures", name); + } + + return File.ReadAllText(candidate); } private static string NormalizeLineEndings(string value) diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json index bef21a2d..517f2a55 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json @@ -3,19 +3,244 @@ "advisoryKey": "APSB25-85", "affectedPackages": [ { - "identifier": "Security update available for Adobe Acrobat Reader", - "platform": null, + "identifier": "Acrobat DC", + "platform": "Windows", "provenance": [ { - "kind": "parser", + "kind": "affected", "recordedAt": "2025-09-10T00:00:00+00:00", "source": "vndr-adobe", - "value": "APSB25-85" + "value": "Acrobat DC:Windows" + } + ], + "statuses": [ + { + "provenance": { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:Windows" + }, + "status": "fixed" } ], - "statuses": [], "type": "vendor", - "versionRanges": [] + "versionRanges": [ + { + "fixedVersion": "25.001.20680", + "introducedVersion": null, + "lastAffectedVersion": "25.001.20672", + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "25.1.20680", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "25.1.20672", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Continuous", + "adobe.platform": "Windows", + "adobe.affected.raw": "25.001.20672 and earlier", + "adobe.updated.raw": "25.001.20680", + "adobe.priority": "Priority 2", + "adobe.availability": "Available" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:Windows" + }, + "rangeExpression": "25.001.20672 and earlier", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Acrobat DC", + "platform": "macOS", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:macOS" + } + ], + "statuses": [ + { + "provenance": { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:macOS" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "25.001.20678", + "introducedVersion": null, + "lastAffectedVersion": "25.001.20668", + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "25.1.20678", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "25.1.20668", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Continuous", + "adobe.platform": "macOS", + "adobe.affected.raw": "25.001.20668 and earlier", + "adobe.updated.raw": "25.001.20678", + "adobe.priority": "Priority 2", + "adobe.availability": "Available" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat DC:macOS" + }, + "rangeExpression": "25.001.20668 and earlier", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Acrobat Reader DC", + "platform": "Windows", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:Windows" + } + ], + "statuses": [ + { + "provenance": { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:Windows" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "25.001.20680", + "introducedVersion": null, + "lastAffectedVersion": "25.001.20672", + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "25.1.20680", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "25.1.20672", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Continuous", + "adobe.platform": "Windows", + "adobe.affected.raw": "25.001.20672 and earlier", + "adobe.updated.raw": "25.001.20680", + "adobe.priority": "Priority 2", + "adobe.availability": "Available" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:Windows" + }, + "rangeExpression": "25.001.20672 and earlier", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Acrobat Reader DC", + "platform": "macOS", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:macOS" + } + ], + "statuses": [ + { + "provenance": { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:macOS" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "25.001.20678", + "introducedVersion": null, + "lastAffectedVersion": "25.001.20668", + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "25.1.20678", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "25.1.20668", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Continuous", + "adobe.platform": "macOS", + "adobe.affected.raw": "25.001.20668 and earlier", + "adobe.updated.raw": "25.001.20678", + "adobe.priority": "Priority 2", + "adobe.availability": "Available" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Acrobat Reader DC:macOS" + }, + "rangeExpression": "25.001.20668 and earlier", + "rangeKind": "vendor" + } + ] } ], "aliases": [ @@ -56,19 +281,124 @@ "advisoryKey": "APSB25-87", "affectedPackages": [ { - "identifier": "Security update available for Adobe Premiere Pro", - "platform": null, + "identifier": "Premiere Pro", + "platform": "Windows", "provenance": [ { - "kind": "parser", + "kind": "affected", "recordedAt": "2025-09-10T00:00:00+00:00", "source": "vndr-adobe", - "value": "APSB25-87" + "value": "Premiere Pro:Windows" + } + ], + "statuses": [ + { + "provenance": { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:Windows" + }, + "status": "fixed" } ], - "statuses": [], "type": "vendor", - "versionRanges": [] + "versionRanges": [ + { + "fixedVersion": "24.6", + "introducedVersion": null, + "lastAffectedVersion": "24.5", + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "24.6", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "24.5", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Quarterly", + "adobe.platform": "Windows", + "adobe.affected.raw": "24.5 and earlier", + "adobe.updated.raw": "24.6", + "adobe.priority": "Priority 3", + "adobe.availability": "Available" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:Windows" + }, + "rangeExpression": "24.5 and earlier", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Premiere Pro", + "platform": "macOS", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:macOS" + } + ], + "statuses": [ + { + "provenance": { + "kind": "affected", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:macOS" + }, + "status": "fixed" + } + ], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "24.6", + "introducedVersion": null, + "lastAffectedVersion": "24.5", + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "24.6", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": "24.5", + "lastAffectedInclusive": true + }, + "vendorExtensions": { + "adobe.track": "Quarterly", + "adobe.platform": "macOS", + "adobe.affected.raw": "24.5 and earlier", + "adobe.updated.raw": "24.6", + "adobe.priority": "Priority 3", + "adobe.availability": "Available" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "Premiere Pro:macOS" + }, + "rangeExpression": "24.5 and earlier", + "rangeKind": "vendor" + } + ] } ], "aliases": [ diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html index fce3db2c..e75ee447 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html @@ -4,7 +4,69 @@ <title>APSB25-85 -

APSB25-85 Security update available for Adobe Acrobat Reader

+

APSB25-85: Security update available for Adobe Acrobat Reader

Date published: September 9, 2025

+ +

Affected Versions

+ + + + + + + + + + + + + + + + + + + +
ProductTrackAffected VersionsPlatform
Acrobat DCContinuous +

Win - 25.001.20672 and earlier

+

Mac - 25.001.20668 and earlier

+ 		Windows & macOS
Acrobat Reader DCContinuous +

Win - 25.001.20672 and earlier

+

Mac - 25.001.20668 and earlier

+ 		Windows & macOS
+ +

Updated Versions

+ + + + + + + + + + + + + + + + + + + + + + + + + +
ProductTrackUpdated VersionsPlatformPriorityAvailability
Acrobat DCContinuous +

Win - 25.001.20680

+

Mac - 25.001.20678

+ 		Windows & macOSPriority 2Available
Acrobat Reader DCContinuous +

Win - 25.001.20680

+

Mac - 25.001.20678

+ 		Windows & macOSPriority 2Available
diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html index eef7154a..993481c9 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html @@ -4,7 +4,49 @@ APSB25-87 -

APSB25-87 Security update available for Adobe Premiere Pro

+

APSB25-87: Security update available for Adobe Premiere Pro

Date published: September 8, 2025

+ +

Affected Versions

+ + + + + + + + + + + + + +
ProductTrackAffected VersionsPlatform
Premiere ProQuarterly +

Win - 24.5 and earlier

+

Mac - 24.5 and earlier

+ 		Windows & macOS
+ +

Updated Versions

+ + + + + + + + + + + + + + + + + +
ProductTrackUpdated VersionsPlatformPriorityAvailability
Premiere ProQuarterly +

Win - 24.6

+

Mac - 24.6

+ 		Windows & macOSPriority 3Available
diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj index d05dbb40..0704eba9 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj @@ -11,7 +11,7 @@ - - + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md index 29038dd5..0a467241 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md @@ -20,10 +20,9 @@ Adobe PSIRT connector ingesting APSB/APA advisories; authoritative for Adobe pro In: PSIRT ingestion, aliases, affected plus fixedBy, psirt_flags, watermark/resume. Out: signing, package artifact downloads, non-Adobe product truth. ## Observability & security expectations -- Metrics: adobe.fetch.count, adobe.parse.fail, adobe.map.affected_count, adobe.cursor.moves. +- Metrics: SourceDiagnostics produces `feedser.source.http.*` counters/histograms tagged `feedser.source=adobe`; operators filter on that tag to monitor fetch counts, parse failures, map affected counts, and cursor movement without bespoke metric names. - Logs: advisory ids, product counts, extraction timings; hosts allowlisted; no secret logging. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Adobe.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs index 0745bf37..4ab478de 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs @@ -3,6 +3,8 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Json.Schema; @@ -10,6 +12,7 @@ using MongoDB.Bson; using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Common.Fetch; using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Common.Packages; using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; using StellaOps.Feedser.Source.Vndr.Adobe.Internal; using StellaOps.Feedser.Storage.Mongo; @@ -42,7 +45,7 @@ public sealed class AdobeConnector : IFeedConnector private static readonly JsonSerializerOptions SerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; public AdobeConnector( @@ -76,6 +79,209 @@ public sealed class AdobeConnector : IFeedConnector _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } + private static IReadOnlyList BuildStatuses(AdobeProductEntry product, AdvisoryProvenance provenance) + { + if (!TryResolveAvailabilityStatus(product.Availability, out var status)) + { + return Array.Empty(); + } + + return new[] { new AffectedPackageStatus(status, provenance) }; + } + + private static bool TryResolveAvailabilityStatus(string? availability, out string status) + { + status = string.Empty; + if (string.IsNullOrWhiteSpace(availability)) + { + return false; + } + + var trimmed = availability.Trim(); + + if (AffectedPackageStatusCatalog.TryNormalize(trimmed, out var normalized)) + { + status = normalized; + return true; + } + + var token = SanitizeStatusToken(trimmed); + if (token.Length == 0) + { + return false; + } + + if (AvailabilityStatusMap.TryGetValue(token, out var mapped)) + { + status = mapped; + return true; + } + + return false; + } + + private static string SanitizeStatusToken(string value) + { + var buffer = new char[value.Length]; + var index = 0; + + foreach (var ch in value) + { + if (char.IsLetterOrDigit(ch)) + { + buffer[index++] = char.ToLowerInvariant(ch); + } + } + + return index == 0 ? string.Empty : new string(buffer, 0, index); + } + + private static readonly Dictionary AvailabilityStatusMap = new(StringComparer.Ordinal) + { + ["available"] = AffectedPackageStatusCatalog.Fixed, + ["availabletoday"] = AffectedPackageStatusCatalog.Fixed, + ["availablenow"] = AffectedPackageStatusCatalog.Fixed, + ["updateavailable"] = AffectedPackageStatusCatalog.Fixed, + ["patchavailable"] = AffectedPackageStatusCatalog.Fixed, + ["fixavailable"] = AffectedPackageStatusCatalog.Fixed, + ["mitigationavailable"] = AffectedPackageStatusCatalog.Mitigated, + ["workaroundavailable"] = AffectedPackageStatusCatalog.Mitigated, + ["mitigationprovided"] = AffectedPackageStatusCatalog.Mitigated, + ["workaroundprovided"] = AffectedPackageStatusCatalog.Mitigated, + ["planned"] = AffectedPackageStatusCatalog.Pending, + ["updateplanned"] = AffectedPackageStatusCatalog.Pending, + ["plannedupdate"] = AffectedPackageStatusCatalog.Pending, + ["scheduled"] = AffectedPackageStatusCatalog.Pending, + ["scheduledupdate"] = AffectedPackageStatusCatalog.Pending, + ["pendingavailability"] = AffectedPackageStatusCatalog.Pending, + ["pendingupdate"] = AffectedPackageStatusCatalog.Pending, + ["pendingfix"] = AffectedPackageStatusCatalog.Pending, + ["notavailable"] = AffectedPackageStatusCatalog.Unknown, + ["unavailable"] = AffectedPackageStatusCatalog.Unknown, + ["notcurrentlyavailable"] = AffectedPackageStatusCatalog.Unknown, + ["notapplicable"] = AffectedPackageStatusCatalog.NotApplicable, + }; + + private AffectedVersionRange? BuildVersionRange(AdobeProductEntry product, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(product.AffectedVersion) && string.IsNullOrWhiteSpace(product.UpdatedVersion)) + { + return null; + } + + var key = string.IsNullOrWhiteSpace(product.Platform) + ? product.Product + : $"{product.Product}:{product.Platform}"; + + var provenance = new AdvisoryProvenance(SourceName, "range", key, recordedAt); + + var extensions = new Dictionary(StringComparer.Ordinal); + AddExtension(extensions, "adobe.track", product.Track); + AddExtension(extensions, "adobe.platform", product.Platform); + AddExtension(extensions, "adobe.affected.raw", product.AffectedVersion); + AddExtension(extensions, "adobe.updated.raw", product.UpdatedVersion); + AddExtension(extensions, "adobe.priority", product.Priority); + AddExtension(extensions, "adobe.availability", product.Availability); + + var lastAffected = ExtractVersionNumber(product.AffectedVersion); + var fixedVersion = ExtractVersionNumber(product.UpdatedVersion); + + var primitives = BuildRangePrimitives(lastAffected, fixedVersion, extensions); + + return new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: fixedVersion, + lastAffectedVersion: lastAffected, + rangeExpression: product.AffectedVersion ?? product.UpdatedVersion, + provenance: provenance, + primitives: primitives); + } + + private static RangePrimitives? BuildRangePrimitives(string? lastAffected, string? fixedVersion, Dictionary extensions) + { + var semVer = BuildSemVerPrimitive(lastAffected, fixedVersion); + + if (semVer is null && extensions.Count == 0) + { + return null; + } + + return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions); + } + + private static SemVerPrimitive? BuildSemVerPrimitive(string? lastAffected, string? fixedVersion) + { + var fixedNormalized = NormalizeSemVer(fixedVersion); + var lastNormalized = NormalizeSemVer(lastAffected); + + if (fixedNormalized is null && lastNormalized is null) + { + return null; + } + + return new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: fixedNormalized, + FixedInclusive: false, + LastAffected: lastNormalized, + LastAffectedInclusive: true, + ConstraintExpression: null); + } + + private static string? NormalizeSemVer(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + if (PackageCoordinateHelper.TryParseSemVer(trimmed, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized)) + { + return normalized; + } + + if (Version.TryParse(trimmed, out var parsed)) + { + if (parsed.Build >= 0 && parsed.Revision >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}"; + } + + if (parsed.Build >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}"; + } + + return $"{parsed.Major}.{parsed.Minor}"; + } + + return null; + } + + private static string? ExtractVersionNumber(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + var match = VersionPattern.Match(text); + return match.Success ? match.Value : null; + } + + private static void AddExtension(IDictionary extensions, string key, string? value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + extensions[key] = value.Trim(); + } + } + + private static readonly Regex VersionPattern = new("\\d+(?:\\.\\d+)+", RegexOptions.Compiled); + public string SourceName => VndrAdobeConnectorPlugin.SourceName; public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) @@ -458,14 +664,8 @@ public sealed class AdobeConnector : IFeedConnector .Select(t => t.Reference) .ToArray(); - var affected = dto.Products.Select(product => - new AffectedPackage( - AffectedPackageTypes.Vendor, - NormalizeIdentifier(product), - platform: null, - versionRanges: Array.Empty(), - statuses: Array.Empty(), - provenance: new[] { provenance })) + var affected = dto.Products + .Select(product => BuildPackage(product, recordedAt)) .ToArray(); var aliases = aliasSet @@ -491,13 +691,30 @@ public sealed class AdobeConnector : IFeedConnector new[] { provenance }); } - private static string NormalizeIdentifier(string product) + private AffectedPackage BuildPackage(AdobeProductEntry product, DateTimeOffset recordedAt) { - if (string.IsNullOrWhiteSpace(product)) - { - return "Adobe Product"; - } + var identifier = string.IsNullOrWhiteSpace(product.Product) + ? "Adobe Product" + : product.Product.Trim(); - return product.Trim(); + var platform = string.IsNullOrWhiteSpace(product.Platform) ? null : product.Platform; + + var provenance = new AdvisoryProvenance( + SourceName, + "affected", + string.IsNullOrWhiteSpace(platform) ? identifier : $"{identifier}:{platform}", + recordedAt); + + var range = BuildVersionRange(product, recordedAt); + var ranges = range is null ? Array.Empty() : new[] { range }; + var statuses = BuildStatuses(product, provenance); + + return new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + platform, + ranges, + statuses, + new[] { provenance }); } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs index 40209c9c..6703fed6 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs @@ -8,7 +8,7 @@ internal sealed record AdobeBulletinDto( string AdvisoryId, string Title, DateTimeOffset Published, - IReadOnlyList Products, + IReadOnlyList Products, IReadOnlyList Cves, string DetailUrl, string? Summary) @@ -17,7 +17,7 @@ internal sealed record AdobeBulletinDto( string advisoryId, string title, DateTimeOffset published, - IEnumerable? products, + IEnumerable? products, IEnumerable? cves, Uri detailUri, string? summary) @@ -26,11 +26,15 @@ internal sealed record AdobeBulletinDto( ArgumentException.ThrowIfNullOrEmpty(title); ArgumentNullException.ThrowIfNull(detailUri); - var productList = products?.Where(static p => !string.IsNullOrWhiteSpace(p)) - .Select(static p => p.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase) - .ToList() ?? new List(); + var productList = products? + .Where(static p => !string.IsNullOrWhiteSpace(p.Product)) + .Select(static p => p with { Product = p.Product.Trim() }) + .Distinct(AdobeProductEntryComparer.Instance) + .OrderBy(static p => p.Product, StringComparer.OrdinalIgnoreCase) + .ThenBy(static p => p.Platform, StringComparer.OrdinalIgnoreCase) + .ThenBy(static p => p.Track, StringComparer.OrdinalIgnoreCase) + .ToList() + ?? new List(); var cveList = cves?.Where(static c => !string.IsNullOrWhiteSpace(c)) .Select(static c => c.Trim().ToUpperInvariant()) @@ -48,3 +52,51 @@ internal sealed record AdobeBulletinDto( string.IsNullOrWhiteSpace(summary) ? null : summary.Trim()); } } + +internal sealed record AdobeProductEntry( + string Product, + string Track, + string Platform, + string? AffectedVersion, + string? UpdatedVersion, + string? Priority, + string? Availability); + +internal sealed class AdobeProductEntryComparer : IEqualityComparer +{ + public static AdobeProductEntryComparer Instance { get; } = new(); + + public bool Equals(AdobeProductEntry? x, AdobeProductEntry? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Track, y.Track, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Platform, y.Platform, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.AffectedVersion, y.AffectedVersion, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.UpdatedVersion, y.UpdatedVersion, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Priority, y.Priority, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Availability, y.Availability, StringComparison.OrdinalIgnoreCase); + } + + public int GetHashCode(AdobeProductEntry obj) + { + var hash = new HashCode(); + hash.Add(obj.Product, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Track, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Platform, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.AffectedVersion, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.UpdatedVersion, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Priority, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Availability, StringComparer.OrdinalIgnoreCase); + return hash.ToHashCode(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs index 0ff1b2d9..7d685d3a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.Linq; using System.Text.RegularExpressions; using AngleSharp.Dom; +using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; @@ -26,7 +27,7 @@ internal static class AdobeDetailParser var published = metadata.PublishedUtc ?? TryExtractPublished(document) ?? DateTimeOffset.UtcNow; var cves = ExtractCves(document.Body?.TextContent ?? string.Empty); - var products = ExtractProducts(title, document); + var products = ExtractProductEntries(title, document); return AdobeBulletinDto.Create( metadata.AdvisoryId, @@ -38,47 +39,250 @@ internal static class AdobeDetailParser summary); } - private static IEnumerable ExtractCves(string text) + private static IReadOnlyList ExtractCves(string text) { if (string.IsNullOrWhiteSpace(text)) + { + return Array.Empty(); + } + + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in CveRegex.Matches(text)) + { + if (!string.IsNullOrWhiteSpace(match.Value)) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + + return set.Count == 0 ? Array.Empty() : set.OrderBy(static cve => cve, StringComparer.Ordinal).ToArray(); + } + + private static IReadOnlyList ExtractProductEntries(string title, IDocument document) + { + var builders = new Dictionary(AdobeProductKeyComparer.Instance); + + foreach (var builder in ParseAffectedTable(document)) + { + builders[builder.Key] = builder; + } + + foreach (var updated in ParseUpdatedTable(document)) + { + if (builders.TryGetValue(updated.Key, out var builder)) + { + builder.UpdatedVersion ??= updated.UpdatedVersion; + builder.Priority ??= updated.Priority; + builder.Availability ??= updated.Availability; + } + else + { + builders[updated.Key] = updated; + } + } + + if (builders.Count == 0 && !string.IsNullOrWhiteSpace(title)) + { + var fallback = new AdobeProductEntryBuilder( + NormalizeWhitespace(title), + string.Empty, + string.Empty) + { + AffectedVersion = null, + UpdatedVersion = null, + Priority = null, + Availability = null + }; + + builders[fallback.Key] = fallback; + } + + return builders.Values + .Select(static builder => builder.ToEntry()) + .ToList(); + } + + private static IEnumerable ParseAffectedTable(IDocument document) + { + var table = FindTableByHeader(document, "Affected Versions"); + if (table is null) { yield break; } - foreach (Match match in CveRegex.Matches(text)) + foreach (var row in table.Rows.Skip(1)) { - yield return match.Value; + var cells = row.Cells; + if (cells.Length < 3) + { + continue; + } + + var product = NormalizeWhitespace(cells[0]?.TextContent); + var track = NormalizeWhitespace(cells.ElementAtOrDefault(1)?.TextContent); + var platformText = NormalizeWhitespace(cells.ElementAtOrDefault(3)?.TextContent); + + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var affectedCell = cells[2]; + foreach (var line in ExtractLines(affectedCell)) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var (platform, versionText) = SplitPlatformLine(line, platformText); + var builder = new AdobeProductEntryBuilder(product, track, platform) + { + AffectedVersion = versionText + }; + + yield return builder; + } } } - private static IEnumerable ExtractProducts(string title, IDocument document) + private static IEnumerable ParseUpdatedTable(IDocument document) { - var products = new List(); - - if (!string.IsNullOrWhiteSpace(title)) + var table = FindTableByHeader(document, "Updated Versions"); + if (table is null) { - var split = title.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); - if (split.Length == 2) - { - products.Add(split[1]); - } - else if (split.Length == 1) - { - var parts = title.Split(" for ", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); - if (parts.Length == 2) - { - products.Add(parts[1]); - } - } + yield break; } - var tableProducts = document.QuerySelectorAll("table td") - .Select(cell => cell.TextContent?.Trim()) - .Where(text => !string.IsNullOrWhiteSpace(text) && text!.Contains("Adobe", StringComparison.OrdinalIgnoreCase)) - .ToList(); + foreach (var row in table.Rows.Skip(1)) + { + var cells = row.Cells; + if (cells.Length < 3) + { + continue; + } - products.AddRange(tableProducts!); - return products; + var product = NormalizeWhitespace(cells[0]?.TextContent); + var track = NormalizeWhitespace(cells.ElementAtOrDefault(1)?.TextContent); + var platformText = NormalizeWhitespace(cells.ElementAtOrDefault(3)?.TextContent); + var priority = NormalizeWhitespace(cells.ElementAtOrDefault(4)?.TextContent); + var availability = NormalizeWhitespace(cells.ElementAtOrDefault(5)?.TextContent); + + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var updatedCell = cells[2]; + var lines = ExtractLines(updatedCell); + if (lines.Count == 0) + { + lines.Add(updatedCell.TextContent ?? string.Empty); + } + + foreach (var line in lines) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var (platform, versionText) = SplitPlatformLine(line, platformText); + var builder = new AdobeProductEntryBuilder(product, track, platform) + { + UpdatedVersion = versionText, + Priority = priority, + Availability = availability + }; + + yield return builder; + } + } + } + + private static IHtmlTableElement? FindTableByHeader(IDocument document, string headerText) + { + return document + .QuerySelectorAll("table") + .OfType() + .FirstOrDefault(table => table.TextContent.Contains(headerText, StringComparison.OrdinalIgnoreCase)); + } + + private static List ExtractLines(IElement? cell) + { + var lines = new List(); + if (cell is null) + { + return lines; + } + + var paragraphs = cell.QuerySelectorAll("p").Select(static p => p.TextContent).ToArray(); + if (paragraphs.Length > 0) + { + foreach (var paragraph in paragraphs) + { + var normalized = NormalizeWhitespace(paragraph); + if (!string.IsNullOrWhiteSpace(normalized)) + { + lines.Add(normalized); + } + } + + return lines; + } + + var items = cell.QuerySelectorAll("li").Select(static li => li.TextContent).ToArray(); + if (items.Length > 0) + { + foreach (var item in items) + { + var normalized = NormalizeWhitespace(item); + if (!string.IsNullOrWhiteSpace(normalized)) + { + lines.Add(normalized); + } + } + + return lines; + } + + var raw = NormalizeWhitespace(cell.TextContent); + if (!string.IsNullOrWhiteSpace(raw)) + { + lines.AddRange(raw.Split(new[] { '\n' }, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)); + } + + return lines; + } + + private static (string Platform, string? Version) SplitPlatformLine(string line, string? fallbackPlatform) + { + var separatorIndex = line.IndexOf('-', StringComparison.Ordinal); + if (separatorIndex > 0 && separatorIndex < line.Length - 1) + { + var prefix = line[..separatorIndex].Trim(); + var versionText = line[(separatorIndex + 1)..].Trim(); + return (NormalizePlatform(prefix) ?? NormalizePlatform(fallbackPlatform) ?? fallbackPlatform ?? string.Empty, versionText); + } + + return (NormalizePlatform(fallbackPlatform) ?? fallbackPlatform ?? string.Empty, line.Trim()); + } + + private static string? NormalizePlatform(string? platform) + { + if (string.IsNullOrWhiteSpace(platform)) + { + return null; + } + + var trimmed = platform.Trim(); + return trimmed.ToLowerInvariant() switch + { + "win" or "windows" => "Windows", + "mac" or "macos" or "mac os" => "macOS", + "windows & macos" or "windows &  macos" => "Windows & macOS", + _ => trimmed + }; } private static DateTimeOffset? TryExtractPublished(IDocument document) @@ -130,4 +334,72 @@ internal static class AdobeDetailParser return false; } + + private static string NormalizeWhitespace(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var sanitized = value ?? string.Empty; + return string.Join(" ", sanitized.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + } + + private sealed record AdobeProductKey(string Product, string Track, string Platform); + + private sealed class AdobeProductKeyComparer : IEqualityComparer + { + public static AdobeProductKeyComparer Instance { get; } = new(); + + public bool Equals(AdobeProductKey? x, AdobeProductKey? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Track, y.Track, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Platform, y.Platform, StringComparison.OrdinalIgnoreCase); + } + + public int GetHashCode(AdobeProductKey obj) + { + var hash = new HashCode(); + hash.Add(obj.Product, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Track, StringComparer.OrdinalIgnoreCase); + hash.Add(obj.Platform, StringComparer.OrdinalIgnoreCase); + return hash.ToHashCode(); + } + } + + private sealed class AdobeProductEntryBuilder + { + public AdobeProductEntryBuilder(string product, string track, string platform) + { + Product = NormalizeWhitespace(product); + Track = NormalizeWhitespace(track); + Platform = NormalizeWhitespace(platform); + } + + public AdobeProductKey Key => new(Product, Track, Platform); + + public string Product { get; } + public string Track { get; } + public string Platform { get; } + + public string? AffectedVersion { get; set; } + public string? UpdatedVersion { get; set; } + public string? Priority { get; set; } + public string? Availability { get; set; } + + public AdobeProductEntry ToEntry() + => new(Product, Track, Platform, AffectedVersion, UpdatedVersion, Priority, Availability); + } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json b/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json index e000460e..0630f2c1 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json @@ -26,8 +26,37 @@ "products": { "type": "array", "items": { - "type": "string", - "minLength": 1 + "type": "object", + "required": [ + "product", + "track", + "platform" + ], + "properties": { + "product": { + "type": "string", + "minLength": 1 + }, + "track": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "affectedVersion": { + "type": ["string", "null"] + }, + "updatedVersion": { + "type": ["string", "null"] + }, + "priority": { + "type": ["string", "null"] + }, + "availability": { + "type": ["string", "null"] + } + }, + "additionalProperties": false } }, "cves": { @@ -44,5 +73,6 @@ "summary": { "type": ["string", "null"] } - } + }, + "additionalProperties": false } diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs index 7f537256..eb3602f2 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs @@ -85,11 +85,11 @@ public sealed class ChromiumConnectorTests : IAsyncLifetime Assert.Equal("Google", psirtFlag!.Vendor); var canonicalJson = CanonicalJsonSerializer.Serialize(advisory).Trim(); - var snapshotPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Chromium", "Fixtures", "chromium-advisory.snapshot.json"); + var snapshotPath = ResolveFixturePath("chromium-advisory.snapshot.json"); var expected = File.ReadAllText(snapshotPath).Trim(); if (!string.Equals(expected, canonicalJson, StringComparison.Ordinal)) { - var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Chromium", "Fixtures", "chromium-advisory.actual.json"); + var actualPath = ResolveFixturePath("chromium-advisory.actual.json"); File.WriteAllText(actualPath, canonicalJson); } Assert.Equal(expected, canonicalJson); @@ -331,10 +331,22 @@ public sealed class ChromiumConnectorTests : IAsyncLifetime private static string ReadFixture(string filename) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Chromium", "Fixtures", filename); + var path = ResolveFixturePath(filename); return File.ReadAllText(path); } + private static string ResolveFixturePath(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Vndr", "Chromium", "Fixtures", filename); + if (File.Exists(primary)) + { + return primary; + } + + return Path.Combine(baseDirectory, "Chromium", "Fixtures", filename); + } + public Task InitializeAsync() => Task.CompletedTask; public async Task DisposeAsync() diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs index 4567ef56..2c15ba7e 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs @@ -38,9 +38,9 @@ public sealed class ChromiumMapperTests Assert.Equal( new[] { - "https://chromium.example/stable-update.html", "https://chromium.example/ref1", "https://chromium.example/ref2", + "https://chromium.example/stable-update.html", }, referenceUrls); } diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json index bb737de8..43f86c3c 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json @@ -1 +1 @@ -{"advisoryKey":"chromium/post/stable-channel-update-for-desktop","affectedPackages":[{"identifier":"google:chrome","platform":"android","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.89","introducedVersion":null,"lastAffectedVersion":null,"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome","platform":"linux","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.137","introducedVersion":null,"lastAffectedVersion":null,"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome","platform":"windows-mac","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.138","introducedVersion":null,"lastAffectedVersion":null,"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome:extended-stable","platform":"windows-mac","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.138","introducedVersion":null,"lastAffectedVersion":null,"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]}],"aliases":["CHROMIUM-POST:2024-09-10","CHROMIUM-POST:stable-channel-update-for-desktop","CVE-2024-12345","CVE-2024-22222"],"cvssMetrics":[],"exploitKnown":false,"language":"en","modified":"2024-09-10T17:45:00+00:00","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"published":"2024-09-10T17:30:00+00:00","references":[{"kind":"advisory","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"chromium-blog","summary":null,"url":"https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"},{"kind":"changelog","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"changelog","summary":"log","url":"https://chromium.googlesource.com/chromium/src/+log/128.0.6613.120..128.0.6613.138"},{"kind":"doc","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"doc","summary":"security page","url":"https://chromium.org/Home/chromium-security"},{"kind":"bug","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"bug","summary":"issue tracker","url":"https://issues.chromium.org/issues/123456789"}],"severity":null,"summary":"Stable channel update rolling out to Windows, macOS, Linux.","title":"Stable Channel Update for Desktop"} \ No newline at end of file +{"advisoryKey":"chromium/post/stable-channel-update-for-desktop","affectedPackages":[{"identifier":"google:chrome","platform":"android","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.89","introducedVersion":null,"lastAffectedVersion":null,"primitives":{"evr":null,"nevra":null,"semVer":null,"vendorExtensions":{"chromium.channel":"stable","chromium.platform":"android","chromium.version.raw":"128.0.6613.89","chromium.version.normalized":"128.0.6613.89","chromium.version.major":"128","chromium.version.minor":"0","chromium.version.build":"6613","chromium.version.patch":"89"}},"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome","platform":"linux","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.137","introducedVersion":null,"lastAffectedVersion":null,"primitives":{"evr":null,"nevra":null,"semVer":null,"vendorExtensions":{"chromium.channel":"stable","chromium.platform":"linux","chromium.version.raw":"128.0.6613.137","chromium.version.normalized":"128.0.6613.137","chromium.version.major":"128","chromium.version.minor":"0","chromium.version.build":"6613","chromium.version.patch":"137"}},"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome","platform":"windows-mac","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.138","introducedVersion":null,"lastAffectedVersion":null,"primitives":{"evr":null,"nevra":null,"semVer":null,"vendorExtensions":{"chromium.channel":"stable","chromium.platform":"windows-mac","chromium.version.raw":"128.0.6613.138","chromium.version.normalized":"128.0.6613.138","chromium.version.major":"128","chromium.version.minor":"0","chromium.version.build":"6613","chromium.version.patch":"138"}},"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome:extended-stable","platform":"windows-mac","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.138","introducedVersion":null,"lastAffectedVersion":null,"primitives":{"evr":null,"nevra":null,"semVer":null,"vendorExtensions":{"chromium.channel":"extended-stable","chromium.platform":"windows-mac","chromium.version.raw":"128.0.6613.138","chromium.version.normalized":"128.0.6613.138","chromium.version.major":"128","chromium.version.minor":"0","chromium.version.build":"6613","chromium.version.patch":"138"}},"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]}],"aliases":["CHROMIUM-POST:2024-09-10","CHROMIUM-POST:stable-channel-update-for-desktop","CVE-2024-12345","CVE-2024-22222"],"cvssMetrics":[],"exploitKnown":false,"language":"en","modified":"2024-09-10T17:45:00+00:00","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"published":"2024-09-10T17:30:00+00:00","references":[{"kind":"advisory","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"chromium-blog","summary":null,"url":"https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"},{"kind":"changelog","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"changelog","summary":"log","url":"https://chromium.googlesource.com/chromium/src/+log/128.0.6613.120..128.0.6613.138"},{"kind":"doc","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"doc","summary":"security page","url":"https://chromium.org/Home/chromium-security"},{"kind":"bug","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"bug","summary":"issue tracker","url":"https://issues.chromium.org/issues/123456789"}],"severity":null,"summary":"Stable channel update rolling out to Windows, macOS, Linux.","title":"Stable Channel Update for Desktop"} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md index 62ed5040..f9a7ce18 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md @@ -20,10 +20,9 @@ Chromium/Chrome vendor feed connector parsing Stable Channel Update posts; autho In: vendor advisory mapping, fixed version emission per platform, psirt_flags vendor context. Out: OS distro packaging semantics; bug bounty details beyond references. ## Observability & security expectations -- Metrics: chromium.fetch.items, chromium.parse.fail, chromium.map.affected_count. +- Metrics: SourceDiagnostics exports the shared `feedser.source.http.*` counters/histograms tagged `feedser.source=chromium`, enabling dashboards to observe fetch volumes, parse failures, and map affected counts via tag filters. - Logs: post slugs, version extracted, platform coverage, timing; allowlist blog host. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Chromium.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs index 843f0dbb..20acae3e 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Common.Fetch; using StellaOps.Feedser.Source.Common.Json; @@ -82,6 +83,7 @@ public sealed class ChromiumConnector : IFeedConnector var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var now = _timeProvider.GetUtcNow(); var (windowStart, windowEnd) = CalculateWindow(cursor, now); + ProvenanceDiagnostics.ReportResumeWindow(SourceName, windowStart, _logger); IReadOnlyList feedEntries; _diagnostics.FetchAttempt(); diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs index 957b58fa..dc557eae 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Globalization; using StellaOps.Feedser.Models; using StellaOps.Feedser.Storage.Mongo.PsirtFlags; @@ -106,7 +107,8 @@ internal static class ChromiumMapper fixedVersion: version.Version, lastAffectedVersion: null, rangeExpression: null, - provenance); + provenance, + primitives: BuildRangePrimitives(version)); yield return new AffectedPackage( AffectedPackageTypes.Vendor, @@ -117,4 +119,56 @@ internal static class ChromiumMapper provenance: new[] { provenance }); } } + + private static RangePrimitives? BuildRangePrimitives(ChromiumVersionInfo version) + { + var extensions = new Dictionary(StringComparer.Ordinal); + AddExtension(extensions, "chromium.channel", version.Channel); + AddExtension(extensions, "chromium.platform", version.Platform); + AddExtension(extensions, "chromium.version.raw", version.Version); + + if (Version.TryParse(version.Version, out var parsed)) + { + AddExtension(extensions, "chromium.version.normalized", BuildNormalizedVersion(parsed)); + extensions["chromium.version.major"] = parsed.Major.ToString(CultureInfo.InvariantCulture); + extensions["chromium.version.minor"] = parsed.Minor.ToString(CultureInfo.InvariantCulture); + + if (parsed.Build >= 0) + { + extensions["chromium.version.build"] = parsed.Build.ToString(CultureInfo.InvariantCulture); + } + + if (parsed.Revision >= 0) + { + extensions["chromium.version.patch"] = parsed.Revision.ToString(CultureInfo.InvariantCulture); + } + } + + return extensions.Count == 0 ? null : new RangePrimitives(null, null, null, extensions); + } + + private static string BuildNormalizedVersion(Version version) + { + if (version.Build >= 0 && version.Revision >= 0) + { + return $"{version.Major}.{version.Minor}.{version.Build}.{version.Revision}"; + } + + if (version.Build >= 0) + { + return $"{version.Major}.{version.Minor}.{version.Build}"; + } + + return $"{version.Major}.{version.Minor}"; + } + + private static void AddExtension(Dictionary extensions, string key, string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + extensions[key] = value.Trim(); + } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md index b560dfbb..b91b6f55 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md @@ -11,6 +11,7 @@ | CH7 | Stabilize resume integration (preserve pending docs across provider instances) | QA | DONE | Storage.Mongo | Resume integration test exercises pending docs across providers via shared Mongo. | | CH8 | Mark failed parse documents | Conn | DONE | Storage.Mongo | Parse pipeline marks failures; unit tests assert status transitions. | | CH9 | Reference dedupe & ordering | Conn | DONE | Models | Mapper groups references by URL and sorts deterministically. | +| CH10 | Range primitives + provenance instrumentation | Conn | DONE | Models, Storage.Mongo | Vendor primitives + logging in place, resume metrics updated, snapshots refreshed. | ## Changelog - YYYY-MM-DD: Created. diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json index 9d60f230..785aa50f 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json @@ -1,9 +1,155 @@ [ { "advisoryKey": "oracle/cpuapr2024-01-html", - "affectedPackages": [], + "affectedPackages": [ + { + "identifier": "Oracle GraalVM for JDK::Libraries", + "platform": "Libraries", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle GraalVM for JDK::Libraries" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle GraalVM for JDK", + "oracle.productRaw": "Oracle Java SE, Oracle GraalVM for JDK", + "oracle.component": "Libraries", + "oracle.componentRaw": "Libraries", + "oracle.segmentVersions": "21.3.8, 22.0.0", + "oracle.supportedVersions": "Oracle Java SE: 8u401, 11.0.22; Oracle GraalVM for JDK: 21.3.8, 22.0.0", + "oracle.rangeExpression": "21.3.8, 22.0.0 (notes: See Note A for mitigation)", + "oracle.baseExpression": "21.3.8, 22.0.0", + "oracle.notes": "See Note A for mitigation", + "oracle.versionTokens": "21.3.8|22.0.0", + "oracle.versionTokens.normalized": "21.3.8|22.0.0" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle GraalVM for JDK::Libraries" + }, + "rangeExpression": "21.3.8, 22.0.0 (notes: See Note A for mitigation)", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Oracle Java SE::Hotspot", + "platform": "Hotspot", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Java SE::Hotspot" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "8u401", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle Java SE", + "oracle.productRaw": "Oracle Java SE", + "oracle.component": "Hotspot", + "oracle.componentRaw": "Hotspot", + "oracle.segmentVersions": "Oracle Java SE: 8u401, 11.0.22", + "oracle.supportedVersions": "Oracle Java SE: 8u401, 11.0.22", + "oracle.rangeExpression": "Oracle Java SE: 8u401, 11.0.22 (notes: Fixed in 8u401 Patch 123456)", + "oracle.baseExpression": "Oracle Java SE: 8u401, 11.0.22", + "oracle.notes": "Fixed in 8u401 Patch 123456", + "oracle.fixedVersion": "8u401", + "oracle.patchNumber": "123456", + "oracle.versionTokens": "Oracle Java SE: 8u401|11.0.22", + "oracle.versionTokens.normalized": "11.0.22" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Java SE::Hotspot" + }, + "rangeExpression": "Oracle Java SE: 8u401, 11.0.22 (notes: Fixed in 8u401 Patch 123456)", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Oracle Java SE::Libraries", + "platform": "Libraries", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Java SE::Libraries" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle Java SE", + "oracle.productRaw": "Oracle Java SE, Oracle GraalVM for JDK", + "oracle.component": "Libraries", + "oracle.componentRaw": "Libraries", + "oracle.segmentVersions": "8u401, 11.0.22", + "oracle.supportedVersions": "Oracle Java SE: 8u401, 11.0.22; Oracle GraalVM for JDK: 21.3.8, 22.0.0", + "oracle.rangeExpression": "8u401, 11.0.22 (notes: See Note A for mitigation)", + "oracle.baseExpression": "8u401, 11.0.22", + "oracle.notes": "See Note A for mitigation", + "oracle.versionTokens": "8u401|11.0.22", + "oracle.versionTokens.normalized": "11.0.22" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Java SE::Libraries" + }, + "rangeExpression": "8u401, 11.0.22 (notes: See Note A for mitigation)", + "rangeKind": "vendor" + } + ] + } + ], "aliases": [ - "ORACLE:cpuapr2024-01-html" + "CVE-2024-9000", + "CVE-2024-9001", + "ORACLE:CPUAPR2024-01-HTML" ], "cvssMetrics": [], "exploitKnown": false, @@ -12,59 +158,210 @@ "provenance": [ { "kind": "document", - "recordedAt": "2024-04-18T00:01:00+00:00", + "recordedAt": "2024-04-18T00:00:00+00:00", "source": "vndr-oracle", "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + }, + { + "kind": "mapping", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "cpuapr2024-01-html" } ], - "published": "2024-04-18T00:00:00+00:00", + "published": "2024-04-18T12:30:00+00:00", "references": [ { "kind": "reference", "provenance": { - "kind": "document", + "kind": "reference", "recordedAt": "2024-04-18T00:01:00+00:00", "source": "vndr-oracle", - "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + "value": "https://support.oracle.com/kb/123456" }, "sourceTag": null, "summary": null, "url": "https://support.oracle.com/kb/123456" }, { - "kind": "reference", + "kind": "patch", "provenance": { - "kind": "document", + "kind": "reference", "recordedAt": "2024-04-18T00:01:00+00:00", "source": "vndr-oracle", - "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + "value": "https://support.oracle.com/rs?type=doc&id=3010001.1" + }, + "sourceTag": "oracle", + "summary": "Oracle Java SE", + "url": "https://support.oracle.com/rs?type=doc&id=3010001.1" + }, + { + "kind": "patch", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/rs?type=doc&id=3010002.1" + }, + "sourceTag": "oracle", + "summary": "Oracle GraalVM", + "url": "https://support.oracle.com/rs?type=doc&id=3010002.1" + }, + { + "kind": "reference", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://updates.oracle.com/patches/fullpatch" }, "sourceTag": null, "summary": null, - "url": "https://updates.oracle.com/patches/patch01" + "url": "https://updates.oracle.com/patches/fullpatch" }, { "kind": "advisory", "provenance": { - "kind": "document", + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9000" + }, + "sourceTag": "CVE-2024-9000", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9000" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9001" + }, + "sourceTag": "CVE-2024-9001", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9001" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", "recordedAt": "2024-04-18T00:01:00+00:00", "source": "vndr-oracle", "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" }, "sourceTag": "oracle", - "summary": null, + "summary": "cpuapr2024 01 html", "url": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" } ], "severity": null, - "summary": "Oracle CPU April 2024 Advisory 1 Oracle Critical Patch Update Advisory - April 2024 (CPU01) This advisory addresses vulnerabilities in Oracle Database Server. Patch download Support article", + "summary": "Oracle CPU April 2024 Advisory 1 Oracle Critical Patch Update Advisory - April 2024 (CPU01) This advisory addresses vulnerabilities in Oracle Java SE and Oracle GraalVM for JDK. It references CVE-2024-9000 and CVE-2024-9001 with additional remediation steps. Affected Products and Versions Patch Availability Document Oracle Java SE, versions 8u401, 11.0.22 Oracle Java SE Oracle GraalVM for JDK, versions 21.3.8, 22.0.0 Oracle GraalVM CVE ID Product Component Protocol Remote Exploit without Auth.? Base Score Attack Vector Attack Complex Privs Req'd User Interact Scope Confidentiality Integrity Availability Supported Versions Affected Notes CVE-2024-9000 Oracle Java SE Hotspot Multiple Yes 9.8 Network Low None Required Changed High High High Oracle Java SE: 8u401, 11.0.22 Fixed in 8u401 Patch 123456 CVE-2024-9001 Oracle Java SE, Oracle GraalVM for JDK Libraries Multiple Yes 7.5 Network High None Required Changed Medium Medium Medium Oracle Java SE: 8u401, 11.0.22; Oracle GraalVM for JDK: 21.3.8, 22.0.0 See Note A for mitigation Note A: Apply interim update 22.0.0.1 for GraalVM. Patch download Support article", "title": "cpuapr2024 01 html" }, { "advisoryKey": "oracle/cpuapr2024-02-html", - "affectedPackages": [], + "affectedPackages": [ + { + "identifier": "Oracle Database Server::SQL*Plus", + "platform": "SQL*Plus", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Database Server::SQL*Plus" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle Database Server", + "oracle.productRaw": "Oracle Database Server", + "oracle.component": "SQL*Plus", + "oracle.componentRaw": "SQL*Plus", + "oracle.segmentVersions": "Oracle Database Server: 19c, 21c", + "oracle.supportedVersions": "Oracle Database Server: 19c, 21c", + "oracle.rangeExpression": "Oracle Database Server: 19c, 21c (notes: See Note B)", + "oracle.baseExpression": "Oracle Database Server: 19c, 21c", + "oracle.notes": "See Note B", + "oracle.versionTokens": "Oracle Database Server: 19c|21c" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle Database Server::SQL*Plus" + }, + "rangeExpression": "Oracle Database Server: 19c, 21c (notes: See Note B)", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "Oracle WebLogic Server::Console", + "platform": "Console", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle WebLogic Server::Console" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "99999999", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "oracle.product": "Oracle WebLogic Server", + "oracle.productRaw": "Oracle WebLogic Server", + "oracle.component": "Console", + "oracle.componentRaw": "Console", + "oracle.segmentVersions": "Oracle WebLogic Server: 14.1.1.0.0", + "oracle.supportedVersions": "Oracle WebLogic Server: 14.1.1.0.0", + "oracle.rangeExpression": "Oracle WebLogic Server: 14.1.1.0.0 (notes: Patch 99999999 available)", + "oracle.baseExpression": "Oracle WebLogic Server: 14.1.1.0.0", + "oracle.notes": "Patch 99999999 available", + "oracle.fixedVersion": "99999999", + "oracle.patchNumber": "99999999", + "oracle.versionTokens": "Oracle WebLogic Server: 14.1.1.0.0" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "Oracle WebLogic Server::Console" + }, + "rangeExpression": "Oracle WebLogic Server: 14.1.1.0.0 (notes: Patch 99999999 available)", + "rangeKind": "vendor" + } + ] + } + ], "aliases": [ - "ORACLE:cpuapr2024-02-html" + "CVE-2024-9100", + "CVE-2024-9101", + "ORACLE:CPUAPR2024-02-HTML" ], "cvssMetrics": [], "exploitKnown": false, @@ -73,40 +370,94 @@ "provenance": [ { "kind": "document", - "recordedAt": "2024-04-18T00:01:00+00:00", + "recordedAt": "2024-04-18T00:00:00+00:00", "source": "vndr-oracle", "value": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + }, + { + "kind": "mapping", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "cpuapr2024-02-html" } ], - "published": "2024-04-18T00:00:00+00:00", + "published": "2024-04-19T08:15:00+00:00", "references": [ { "kind": "reference", "provenance": { - "kind": "document", + "kind": "reference", "recordedAt": "2024-04-18T00:01:00+00:00", "source": "vndr-oracle", - "value": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + "value": "https://support.oracle.com/kb/789012" }, "sourceTag": null, "summary": null, "url": "https://support.oracle.com/kb/789012" }, + { + "kind": "patch", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/rs?type=doc&id=3010100.1" + }, + "sourceTag": "oracle", + "summary": "Fusion Middleware", + "url": "https://support.oracle.com/rs?type=doc&id=3010100.1" + }, + { + "kind": "patch", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://support.oracle.com/rs?type=doc&id=3010101.1" + }, + "sourceTag": "oracle", + "summary": "Database", + "url": "https://support.oracle.com/rs?type=doc&id=3010101.1" + }, { "kind": "advisory", "provenance": { - "kind": "document", + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9100" + }, + "sourceTag": "CVE-2024-9100", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9100" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9101" + }, + "sourceTag": "CVE-2024-9101", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9101" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", "recordedAt": "2024-04-18T00:01:00+00:00", "source": "vndr-oracle", "value": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" }, "sourceTag": "oracle", - "summary": null, + "summary": "cpuapr2024 02 html", "url": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" } ], "severity": null, - "summary": "Oracle CPU April 2024 Advisory 2 Oracle Security Alert Advisory - April 2024 (CPU02) Mitigations for Oracle WebLogic Server. More details at Support KB .", + "summary": "Oracle CPU April 2024 Advisory 2 Oracle Security Alert Advisory - April 2024 (CPU02) Mitigations for Oracle WebLogic Server and Oracle Database Server. Includes references to CVE-2024-9100 with additional product components. Affected Products and Versions Patch Availability Document Oracle WebLogic Server, versions 14.1.1.0.0 Fusion Middleware Oracle Database Server, versions 19c, 21c Database CVE ID Product Component Protocol Remote Exploit without Auth.? Base Score Attack Vector Attack Complex Privs Req'd User Interact Scope Confidentiality Integrity Availability Supported Versions Affected Notes CVE-2024-9100 Oracle WebLogic Server Console HTTP Yes 8.1 Network Low Low Required Changed High High High Oracle WebLogic Server: 14.1.1.0.0 Patch 99999999 available CVE-2024-9101 Oracle Database Server SQL*Plus Multiple No 5.4 Local Low Low None Unchanged Medium Low Low Oracle Database Server: 19c, 21c See Note B Note B: Customers should review Support Doc 3010101.1 for mitigation guidance. More details at Support KB .", "title": "cpuapr2024 02 html" } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html new file mode 100644 index 00000000..bec8523f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html @@ -0,0 +1,7 @@ + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html new file mode 100644 index 00000000..72d1657b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html @@ -0,0 +1,8 @@ + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html index 7ffd86f7..80b439bc 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html @@ -1,10 +1,107 @@ - Oracle CPU April 2024 Advisory 1 + + Oracle CPU April 2024 Advisory 1 + +

Oracle Critical Patch Update Advisory - April 2024 (CPU01)

-

This advisory addresses vulnerabilities in Oracle Database Server.

+

+ This advisory addresses vulnerabilities in Oracle Java SE and Oracle GraalVM for JDK. + It references CVE-2024-9000 and CVE-2024-9001 with additional remediation steps. +

+ +
+
+ + + + + + + + + + + + + + + + + +
Affected Products and VersionsPatch Availability Document
Oracle Java SE, versions 8u401, 11.0.22Oracle Java SE
Oracle GraalVM for JDK, versions 21.3.8, 22.0.0Oracle GraalVM
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CVE IDProductComponentProtocolRemote Exploit without Auth.?Base ScoreAttack VectorAttack ComplexPrivs Req'dUser InteractScopeConfidentialityIntegrityAvailabilitySupported Versions AffectedNotes
CVE-2024-9000Oracle Java SEHotspotMultipleYes9.8NetworkLowNoneRequiredChangedHighHighHighOracle Java SE: 8u401, 11.0.22Fixed in 8u401 Patch 123456
CVE-2024-9001Oracle Java SE, Oracle GraalVM for JDKLibrariesMultipleYes7.5NetworkHighNoneRequiredChangedMediumMediumMediumOracle Java SE: 8u401, 11.0.22; Oracle GraalVM for JDK: 21.3.8, 22.0.0See Note A for mitigation
+
+
+ +

Note A: Apply interim update 22.0.0.1 for GraalVM.

+ diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html index 764eaaa8..95da3db4 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html @@ -1,8 +1,105 @@ - Oracle CPU April 2024 Advisory 2 + + Oracle CPU April 2024 Advisory 2 + +

Oracle Security Alert Advisory - April 2024 (CPU02)

-

Mitigations for Oracle WebLogic Server.

+

+ Mitigations for Oracle WebLogic Server and Oracle Database Server. + Includes references to CVE-2024-9100 with additional product components. +

+ +
+
+ + + + + + + + + + + + + + + + + +
Affected Products and VersionsPatch Availability Document
Oracle WebLogic Server, versions 14.1.1.0.0Fusion Middleware
Oracle Database Server, versions 19c, 21cDatabase
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CVE IDProductComponentProtocolRemote Exploit without Auth.?Base ScoreAttack VectorAttack ComplexPrivs Req'dUser InteractScopeConfidentialityIntegrityAvailabilitySupported Versions AffectedNotes
CVE-2024-9100Oracle WebLogic ServerConsoleHTTPYes8.1NetworkLowLowRequiredChangedHighHighHighOracle WebLogic Server: 14.1.1.0.0Patch 99999999 available
CVE-2024-9101Oracle Database ServerSQL*PlusMultipleNo5.4LocalLowLowNoneUnchangedMediumLowLowOracle Database Server: 19c, 21cSee Note B
+
+
+ +

Note B: Customers should review Support Doc 3010101.1 for mitigation guidance.

+

More details at Support KB.

diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html new file mode 100644 index 00000000..1895d203 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html @@ -0,0 +1,4 @@ + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs index 5d38b5b3..d18293c6 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs @@ -15,18 +15,22 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; using MongoDB.Driver; using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; using StellaOps.Feedser.Source.Common.Http; using StellaOps.Feedser.Source.Common.Testing; using StellaOps.Feedser.Source.Vndr.Oracle; using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Feedser.Source.Vndr.Oracle.Internal; using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.Storage.Mongo.Advisories; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; using StellaOps.Feedser.Testing; +using Xunit.Abstractions; namespace StellaOps.Feedser.Source.Vndr.Oracle.Tests; @@ -36,15 +40,18 @@ public sealed class OracleConnectorTests : IAsyncLifetime private readonly MongoIntegrationFixture _fixture; private readonly FakeTimeProvider _timeProvider; private readonly CannedHttpMessageHandler _handler; + private readonly ITestOutputHelper _output; private static readonly Uri AdvisoryOne = new("https://www.oracle.com/security-alerts/cpuapr2024-01.html"); private static readonly Uri AdvisoryTwo = new("https://www.oracle.com/security-alerts/cpuapr2024-02.html"); + private static readonly Uri CalendarUri = new("https://www.oracle.com/security-alerts/cpuapr2024.html"); - public OracleConnectorTests(MongoIntegrationFixture fixture) + public OracleConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) { _fixture = fixture; _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 18, 0, 0, 0, TimeSpan.Zero)); _handler = new CannedHttpMessageHandler(); + _output = output; } [Fact] @@ -53,6 +60,14 @@ public sealed class OracleConnectorTests : IAsyncLifetime await using var provider = await BuildServiceProviderAsync(); SeedDetails(); + var calendarFetcher = provider.GetRequiredService(); + var discovered = await calendarFetcher.GetAdvisoryUrisAsync(CancellationToken.None); + _output.WriteLine("Calendar URIs: " + string.Join(", ", discovered.Select(static uri => uri.AbsoluteUri))); + Assert.Equal(2, discovered.Count); + + // Re-seed fixtures because calendar fetch consumes canned responses. + SeedDetails(); + var connector = provider.GetRequiredService(); await connector.FetchAsync(provider, CancellationToken.None); _timeProvider.Advance(TimeSpan.FromMinutes(1)); @@ -61,8 +76,20 @@ public sealed class OracleConnectorTests : IAsyncLifetime var advisoryStore = provider.GetRequiredService(); var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + _output.WriteLine("Advisories fetched: " + string.Join(", ", advisories.Select(static a => a.AdvisoryKey))); + _output.WriteLine($"Advisory count: {advisories.Count}"); Assert.Equal(2, advisories.Count); + var first = advisories.Single(advisory => advisory.AdvisoryKey == "oracle/cpuapr2024-01-html"); + var second = advisories.Single(advisory => advisory.AdvisoryKey == "oracle/cpuapr2024-02-html"); + Assert.Equal(new DateTimeOffset(2024, 4, 18, 12, 30, 0, TimeSpan.Zero), first.Published); + Assert.Equal(new DateTimeOffset(2024, 4, 19, 8, 15, 0, TimeSpan.Zero), second.Published); + Assert.All(advisories, advisory => + { + Assert.True(advisory.Aliases.Any(alias => alias.StartsWith("CVE-", StringComparison.Ordinal)), $"Expected CVE alias for {advisory.AdvisoryKey}"); + Assert.NotEmpty(advisory.AffectedPackages); + }); + var snapshot = SnapshotSerializer.ToSnapshot(advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray()); var expected = ReadFixture("oracle-advisories.snapshot.json"); var normalizedSnapshot = Normalize(snapshot); @@ -70,6 +97,11 @@ public sealed class OracleConnectorTests : IAsyncLifetime if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) { var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Oracle", "Fixtures", "oracle-advisories.actual.json"); + var actualDirectory = Path.GetDirectoryName(actualPath); + if (!string.IsNullOrEmpty(actualDirectory)) + { + Directory.CreateDirectory(actualDirectory); + } File.WriteAllText(actualPath, snapshot); } @@ -77,10 +109,148 @@ public sealed class OracleConnectorTests : IAsyncLifetime var psirtCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); var flags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); + _output.WriteLine("Psirt flags: " + string.Join(", ", flags.Select(doc => doc.GetValue("_id", BsonValue.Create("")).ToString()))); Assert.Equal(2, flags.Count); Assert.All(flags, doc => Assert.Equal("Oracle", doc["vendor"].AsString)); } + [Fact] + public async Task FetchAsync_IdempotentForUnchangedAdvisories() + { + await using var provider = await BuildServiceProviderAsync(); + SeedDetails(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.MapAsync(provider, CancellationToken.None); + + // Second run with unchanged documents should rely on fetch cache. + SeedDetails(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrOracleConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = OracleCursor.FromBson(state!.Cursor); + Assert.Empty(cursor.PendingDocuments); + Assert.Empty(cursor.PendingMappings); + Assert.Equal(2, cursor.FetchCache.Count); + Assert.All(cursor.FetchCache.Values, entry => Assert.False(string.IsNullOrWhiteSpace(entry.Sha256))); + + var documentStore = provider.GetRequiredService(); + var first = await documentStore.FindBySourceAndUriAsync(VndrOracleConnectorPlugin.SourceName, AdvisoryOne.ToString(), CancellationToken.None); + Assert.NotNull(first); + Assert.Equal(DocumentStatuses.Mapped, first!.Status); + + var second = await documentStore.FindBySourceAndUriAsync(VndrOracleConnectorPlugin.SourceName, AdvisoryTwo.ToString(), CancellationToken.None); + Assert.NotNull(second); + Assert.Equal(DocumentStatuses.Mapped, second!.Status); + + var dtoCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Dto); + var dtoCount = await dtoCollection.CountDocumentsAsync(Builders.Filter.Empty); + Assert.Equal(2, dtoCount); + } + + [Fact] + public async Task FetchAsync_ResumeProcessesNewCalendarEntries() + { + await using var provider = await BuildServiceProviderAsync(); + + AddCalendarResponse(CalendarUri, "oracle-calendar-cpuapr2024-single.html"); + AddDetailResponse(AdvisoryOne, "oracle-detail-cpuapr2024-01.html", "\"oracle-001\""); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Single(advisories); + Assert.Equal("oracle/cpuapr2024-01-html", advisories[0].AdvisoryKey); + + _handler.Clear(); + AddCalendarResponse(CalendarUri, "oracle-calendar-cpuapr2024.html"); + AddDetailResponse(AdvisoryOne, "oracle-detail-cpuapr2024-01.html", "\"oracle-001\""); + AddDetailResponse(AdvisoryTwo, "oracle-detail-cpuapr2024-02.html", "\"oracle-002\""); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "oracle/cpuapr2024-02-html"); + } + + [Fact] + public async Task ParseAsync_InvalidDocumentIsQuarantined() + { + await using var provider = await BuildServiceProviderAsync(); + + AddCalendarResponse(CalendarUri, "oracle-calendar-cpuapr2024.html"); + AddDetailResponse(AdvisoryOne, "oracle-detail-invalid.html", "\"oracle-001\""); + AddDetailResponse(AdvisoryTwo, "oracle-detail-cpuapr2024-02.html", "\"oracle-002\""); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var invalidDocument = await documentStore.FindBySourceAndUriAsync(VndrOracleConnectorPlugin.SourceName, AdvisoryOne.ToString(), CancellationToken.None); + Assert.NotNull(invalidDocument); + _output.WriteLine($"Invalid document status: {invalidDocument!.Status}"); + + var rawDoc = await _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Document) + .Find(Builders.Filter.Eq("uri", AdvisoryOne.ToString())) + .FirstOrDefaultAsync(); + if (rawDoc is not null) + { + _output.WriteLine("Raw document: " + rawDoc.ToJson()); + } + + var dtoStore = provider.GetRequiredService(); + var invalidDto = await dtoStore.FindByDocumentIdAsync(invalidDocument.Id, CancellationToken.None); + if (invalidDto is not null) + { + _output.WriteLine("Validation unexpectedly succeeded. DTO: " + invalidDto.Payload.ToJson()); + } + Assert.Equal(DocumentStatuses.Failed, invalidDocument.Status); + Assert.Null(invalidDto); + + var validDocument = await documentStore.FindBySourceAndUriAsync(VndrOracleConnectorPlugin.SourceName, AdvisoryTwo.ToString(), CancellationToken.None); + Assert.NotNull(validDocument); + Assert.Equal(DocumentStatuses.PendingMap, validDocument!.Status); + + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.MapAsync(provider, CancellationToken.None); + + var advisories = await provider.GetRequiredService().GetRecentAsync(10, CancellationToken.None); + Assert.Single(advisories); + Assert.Equal("oracle/cpuapr2024-02-html", advisories[0].AdvisoryKey); + + var psirtCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + var flagCount = await psirtCollection.CountDocumentsAsync(Builders.Filter.Empty); + Assert.Equal(1, flagCount); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrOracleConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = OracleCursor.FromBson(state!.Cursor); + Assert.Empty(cursor.PendingDocuments); + Assert.Empty(cursor.PendingMappings); + } + private async Task BuildServiceProviderAsync() { await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); @@ -101,7 +271,7 @@ public sealed class OracleConnectorTests : IAsyncLifetime services.AddSourceCommon(); services.AddOracleConnector(opts => { - opts.AdvisoryUris = new List { AdvisoryOne, AdvisoryTwo }; + opts.CalendarUris = new List { CalendarUri }; opts.RequestDelay = TimeSpan.Zero; }); @@ -121,10 +291,24 @@ public sealed class OracleConnectorTests : IAsyncLifetime private void SeedDetails() { + AddCalendarResponse(CalendarUri, "oracle-calendar-cpuapr2024.html"); AddDetailResponse(AdvisoryOne, "oracle-detail-cpuapr2024-01.html", "\"oracle-001\""); AddDetailResponse(AdvisoryTwo, "oracle-detail-cpuapr2024-02.html", "\"oracle-002\""); } + private void AddCalendarResponse(Uri uri, string fixture) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + + return response; + }); + } + private void AddDetailResponse(Uri uri, string fixture, string? etag) { _handler.AddResponse(uri, () => @@ -145,8 +329,19 @@ public sealed class OracleConnectorTests : IAsyncLifetime private static string ReadFixture(string filename) { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Oracle", "Fixtures", filename); - return File.ReadAllText(path); + var primary = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Oracle", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(AppContext.BaseDirectory, "Oracle", "Fixtures", filename); + if (File.Exists(fallback)) + { + return File.ReadAllText(fallback); + } + + throw new FileNotFoundException($"Fixture '{filename}' not found in test output.", filename); } private static string Normalize(string value) diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj index b9a6a2ee..2645b4b5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj @@ -12,5 +12,6 @@ + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md index 8130f592..e1919ed4 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md @@ -19,10 +19,9 @@ Oracle PSIRT connector for Critical Patch Updates (CPU) and Security Alerts; aut In: PSIRT authoritative mapping, cycles handling, precedence signaling. Out: signing or patch artifact downloads. ## Observability & security expectations -- Metrics: oracle.fetch.pages, oracle.cpu.cycles, oracle.parse.fail, oracle.map.affected_count. +- Metrics: SourceDiagnostics emits `feedser.source.http.*` counters/histograms tagged `feedser.source=oracle`, so observability dashboards slice on that tag to monitor fetch pages, CPU cycle coverage, parse failures, and map affected counts. - Logs: cycle tags, advisory ids, extraction timings; redact nothing sensitive. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Oracle.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs index 32d1ca6c..f41da348 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs @@ -10,13 +10,15 @@ public sealed class OracleOptions public List AdvisoryUris { get; set; } = new(); + public List CalendarUris { get; set; } = new(); + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromSeconds(1); public void Validate() { - if (AdvisoryUris.Count == 0) + if (AdvisoryUris.Count == 0 && CalendarUris.Count == 0) { - throw new InvalidOperationException("Oracle AdvisoryUris must include at least one URI."); + throw new InvalidOperationException("Oracle connector requires at least one advisory or calendar URI."); } if (AdvisoryUris.Any(uri => uri is null || !uri.IsAbsoluteUri)) @@ -24,6 +26,11 @@ public sealed class OracleOptions throw new InvalidOperationException("All Oracle AdvisoryUris must be absolute URIs."); } + if (CalendarUris.Any(uri => uri is null || !uri.IsAbsoluteUri)) + { + throw new InvalidOperationException("All Oracle CalendarUris must be absolute URIs."); + } + if (RequestDelay < TimeSpan.Zero) { throw new InvalidOperationException("RequestDelay cannot be negative."); diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs new file mode 100644 index 00000000..22220465 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OracleAffectedEntry( + [property: JsonPropertyName("product")] string Product, + [property: JsonPropertyName("component")] string? Component, + [property: JsonPropertyName("supportedVersions")] string? SupportedVersions, + [property: JsonPropertyName("notes")] string? Notes, + [property: JsonPropertyName("cves")] IReadOnlyList CveIds); diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs new file mode 100644 index 00000000..bc59b426 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +public sealed class OracleCalendarFetcher +{ + private static readonly Regex AnchorRegex = new("]+href=\"(?[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private readonly IHttpClientFactory _httpClientFactory; + private readonly OracleOptions _options; + private readonly ILogger _logger; + + public OracleCalendarFetcher( + IHttpClientFactory httpClientFactory, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> GetAdvisoryUrisAsync(CancellationToken cancellationToken) + { + if (_options.CalendarUris.Count == 0) + { + return Array.Empty(); + } + + var discovered = new HashSet(StringComparer.OrdinalIgnoreCase); + var client = _httpClientFactory.CreateClient(OracleOptions.HttpClientName); + + foreach (var calendarUri in _options.CalendarUris) + { + try + { + var content = await client.GetStringAsync(calendarUri, cancellationToken).ConfigureAwait(false); + foreach (var link in ExtractLinks(calendarUri, content)) + { + discovered.Add(link.AbsoluteUri); + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or OperationCanceledException) + { + _logger.LogWarning(ex, "Oracle calendar fetch failed for {Uri}", calendarUri); + } + } + + return discovered + .Select(static uri => new Uri(uri, UriKind.Absolute)) + .OrderBy(static uri => uri.AbsoluteUri, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IEnumerable ExtractLinks(Uri baseUri, string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + yield break; + } + + foreach (Match match in AnchorRegex.Matches(html)) + { + if (!match.Success) + { + continue; + } + + var href = match.Groups["url"].Value?.Trim(); + if (string.IsNullOrEmpty(href)) + { + continue; + } + + if (!Uri.TryCreate(baseUri, href, out var uri) || !uri.IsAbsoluteUri) + { + continue; + } + + yield return uri; + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs index f203a487..27d088a5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs @@ -2,15 +2,21 @@ using System; using System.Collections.Generic; using System.Linq; using MongoDB.Bson; +using StellaOps.Feedser.Storage.Mongo.Documents; namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; internal sealed record OracleCursor( DateTimeOffset? LastProcessed, IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings) + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) { - public static OracleCursor Empty { get; } = new(null, Array.Empty(), Array.Empty()); + private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyFetchCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static OracleCursor Empty { get; } = new(null, EmptyGuidCollection, EmptyGuidCollection, EmptyFetchCache); public BsonDocument ToBsonDocument() { @@ -25,6 +31,17 @@ internal sealed record OracleCursor( document["lastProcessed"] = LastProcessed.Value.UtcDateTime; } + if (FetchCache.Count > 0) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBsonDocument(); + } + + document["fetchCache"] = cacheDocument; + } + return document; } @@ -42,17 +59,39 @@ internal sealed record OracleCursor( return new OracleCursor( lastProcessed, ReadGuidArray(document, "pendingDocuments"), - ReadGuidArray(document, "pendingMappings")); + ReadGuidArray(document, "pendingMappings"), + ReadFetchCache(document)); } public OracleCursor WithLastProcessed(DateTimeOffset? timestamp) => this with { LastProcessed = timestamp }; public OracleCursor WithPendingDocuments(IEnumerable ids) - => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; public OracleCursor WithPendingMappings(IEnumerable ids) - => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; + + public OracleCursor WithFetchCache(IDictionary cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyFetchCache }; + } + + return this with { FetchCache = new Dictionary(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public bool TryGetFetchCache(string key, out OracleFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = OracleFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } private static DateTimeOffset? ParseDate(BsonValue value) => value.BsonType switch @@ -85,4 +124,104 @@ internal sealed record OracleCursor( return result; } + + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var raw) || raw is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) + { + return EmptyFetchCache; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is not BsonDocument entryDocument) + { + continue; + } + + cache[element.Name] = OracleFetchCacheEntry.FromBson(entryDocument); + } + + return cache; + } +} + +internal sealed record OracleFetchCacheEntry(string? Sha256, string? ETag, DateTimeOffset? LastModified) +{ + public static OracleFetchCacheEntry Empty { get; } = new(string.Empty, null, null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["sha256"] = Sha256 ?? string.Empty, + }; + + if (!string.IsNullOrWhiteSpace(ETag)) + { + document["etag"] = ETag; + } + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + return document; + } + + public static OracleFetchCacheEntry FromBson(BsonDocument document) + { + var sha = document.TryGetValue("sha256", out var shaValue) ? shaValue.ToString() : string.Empty; + string? etag = null; + if (document.TryGetValue("etag", out var etagValue) && !etagValue.IsBsonNull) + { + etag = etagValue.ToString(); + } + + DateTimeOffset? lastModified = null; + if (document.TryGetValue("lastModified", out var lastModifiedValue)) + { + lastModified = lastModifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new OracleFetchCacheEntry(sha, etag, lastModified); + } + + public static OracleFetchCacheEntry FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + return new OracleFetchCacheEntry( + document.Sha256 ?? string.Empty, + document.Etag, + document.LastModified?.ToUniversalTime()); + } + + public bool Matches(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + if (!string.IsNullOrEmpty(Sha256) && !string.IsNullOrEmpty(document.Sha256)) + { + return string.Equals(Sha256, document.Sha256, StringComparison.OrdinalIgnoreCase); + } + + if (!string.IsNullOrEmpty(ETag) && !string.IsNullOrEmpty(document.Etag)) + { + return string.Equals(ETag, document.Etag, StringComparison.Ordinal); + } + + if (LastModified.HasValue && document.LastModified.HasValue) + { + return LastModified.Value.ToUniversalTime() == document.LastModified.Value.ToUniversalTime(); + } + + return false; + } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs index a273d71d..580b798f 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs @@ -10,4 +10,7 @@ internal sealed record OracleDto( [property: JsonPropertyName("detailUrl")] string DetailUrl, [property: JsonPropertyName("published")] DateTimeOffset Published, [property: JsonPropertyName("content")] string Content, - [property: JsonPropertyName("references")] IReadOnlyList References); + [property: JsonPropertyName("references")] IReadOnlyList References, + [property: JsonPropertyName("cveIds")] IReadOnlyList CveIds, + [property: JsonPropertyName("affected")] IReadOnlyList Affected, + [property: JsonPropertyName("patchDocuments")] IReadOnlyList PatchDocuments); diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs new file mode 100644 index 00000000..aa4d5b66 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs @@ -0,0 +1,276 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal static class OracleDtoValidator +{ + private const int MaxAdvisoryIdLength = 128; + private const int MaxTitleLength = 512; + private const int MaxContentLength = 200_000; + private const int MaxReferenceCount = 100; + private const int MaxCveCount = 1_024; + private const int MaxAffectedCount = 2_048; + private const int MaxPatchDocumentCount = 512; + private const int MaxProductLength = 512; + private const int MaxComponentLength = 512; + private const int MaxSupportedVersionsLength = 4_096; + private const int MaxNotesLength = 1_024; + private const int MaxPatchTitleLength = 512; + private const int MaxPatchUrlLength = 1_024; + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static bool TryNormalize(OracleDto dto, out OracleDto normalized, out string? failureReason) + { + ArgumentNullException.ThrowIfNull(dto); + + failureReason = null; + normalized = dto; + + var advisoryId = dto.AdvisoryId?.Trim(); + if (string.IsNullOrWhiteSpace(advisoryId)) + { + failureReason = "AdvisoryId is required."; + return false; + } + + if (advisoryId.Length > MaxAdvisoryIdLength) + { + failureReason = $"AdvisoryId exceeds {MaxAdvisoryIdLength} characters."; + return false; + } + + var title = string.IsNullOrWhiteSpace(dto.Title) ? advisoryId : dto.Title.Trim(); + if (title.Length > MaxTitleLength) + { + title = title.Substring(0, MaxTitleLength); + } + + var detailUrlRaw = dto.DetailUrl?.Trim(); + if (string.IsNullOrWhiteSpace(detailUrlRaw) || !Uri.TryCreate(detailUrlRaw, UriKind.Absolute, out var detailUri)) + { + failureReason = "DetailUrl must be an absolute URI."; + return false; + } + + if (!string.Equals(detailUri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(detailUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + failureReason = "DetailUrl must use HTTP or HTTPS."; + return false; + } + + if (dto.Published == default) + { + failureReason = "Published timestamp is required."; + return false; + } + + var published = dto.Published.ToUniversalTime(); + var content = dto.Content?.Trim() ?? string.Empty; + if (string.IsNullOrWhiteSpace(content)) + { + failureReason = "Advisory content is empty."; + return false; + } + + if (content.Length > MaxContentLength) + { + content = content.Substring(0, MaxContentLength); + } + + var references = NormalizeReferences(dto.References); + var cveIds = NormalizeCveIds(dto.CveIds); + var affected = NormalizeAffected(dto.Affected); + var patchDocuments = NormalizePatchDocuments(dto.PatchDocuments); + + normalized = dto with + { + AdvisoryId = advisoryId, + Title = title, + DetailUrl = detailUri.ToString(), + Published = published, + Content = content, + References = references, + CveIds = cveIds, + Affected = affected, + PatchDocuments = patchDocuments, + }; + + return true; + } + + private static IReadOnlyList NormalizeReferences(IReadOnlyList? references) + { + if (references is null || references.Count == 0) + { + return Array.Empty(); + } + + var normalized = new List(Math.Min(references.Count, MaxReferenceCount)); + foreach (var reference in references.Where(static reference => !string.IsNullOrWhiteSpace(reference))) + { + var trimmed = reference.Trim(); + if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) + && (string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + || string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))) + { + normalized.Add(uri.ToString()); + } + + if (normalized.Count >= MaxReferenceCount) + { + break; + } + } + + if (normalized.Count == 0) + { + return Array.Empty(); + } + + return normalized + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList NormalizeCveIds(IReadOnlyList? cveIds) + { + if (cveIds is null || cveIds.Count == 0) + { + return Array.Empty(); + } + + var normalized = new List(Math.Min(cveIds.Count, MaxCveCount)); + foreach (var cve in cveIds.Where(static value => !string.IsNullOrWhiteSpace(value))) + { + var candidate = cve.Trim().ToUpperInvariant(); + if (!CveRegex.IsMatch(candidate)) + { + continue; + } + + normalized.Add(candidate); + if (normalized.Count >= MaxCveCount) + { + break; + } + } + + if (normalized.Count == 0) + { + return Array.Empty(); + } + + return normalized + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList NormalizeAffected(IReadOnlyList? entries) + { + if (entries is null || entries.Count == 0) + { + return Array.Empty(); + } + + var normalized = new List(Math.Min(entries.Count, MaxAffectedCount)); + foreach (var entry in entries) + { + if (entry is null) + { + continue; + } + + var product = TrimToLength(entry.Product, MaxProductLength); + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var component = TrimToNull(entry.Component, MaxComponentLength); + var versions = TrimToNull(entry.SupportedVersions, MaxSupportedVersionsLength); + var notes = TrimToNull(entry.Notes, MaxNotesLength); + var cves = NormalizeCveIds(entry.CveIds); + + normalized.Add(new OracleAffectedEntry(product, component, versions, notes, cves)); + if (normalized.Count >= MaxAffectedCount) + { + break; + } + } + + return normalized.Count == 0 ? Array.Empty() : normalized; + } + + private static IReadOnlyList NormalizePatchDocuments(IReadOnlyList? documents) + { + if (documents is null || documents.Count == 0) + { + return Array.Empty(); + } + + var normalized = new List(Math.Min(documents.Count, MaxPatchDocumentCount)); + foreach (var document in documents) + { + if (document is null) + { + continue; + } + + var product = TrimToLength(document.Product, MaxProductLength); + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var title = TrimToNull(document.Title, MaxPatchTitleLength); + var urlRaw = TrimToLength(document.Url, MaxPatchUrlLength); + if (string.IsNullOrWhiteSpace(urlRaw)) + { + continue; + } + + if (!Uri.TryCreate(urlRaw, UriKind.Absolute, out var uri) + || (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + normalized.Add(new OraclePatchDocument(product, title, uri.ToString())); + if (normalized.Count >= MaxPatchDocumentCount) + { + break; + } + } + + return normalized.Count == 0 ? Array.Empty() : normalized; + } + + private static string TrimToLength(string? value, int maxLength) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var trimmed = value.Trim(); + if (trimmed.Length <= maxLength) + { + return trimmed; + } + + return trimmed[..maxLength]; + } + + private static string? TrimToNull(string? value, int maxLength) + { + var trimmed = TrimToLength(value, maxLength); + return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs index 79f3c5b5..1c87544a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs @@ -1,27 +1,39 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Packages; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; using StellaOps.Feedser.Storage.Mongo.PsirtFlags; namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; internal static class OracleMapper { - public static (Advisory Advisory, PsirtFlagRecord Flag) Map(OracleDto dto, string sourceName, DateTimeOffset recordedAt) + private static readonly Regex FixedVersionRegex = new("(?:Fixed|Fix)\\s+(?:in|available in|for)\\s+(?[A-Za-z0-9._-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex PatchNumberRegex = new("Patch\\s+(?\\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static (Advisory Advisory, PsirtFlagRecord Flag) Map( + OracleDto dto, + DocumentRecord document, + DtoRecord dtoRecord, + string sourceName, + DateTimeOffset mappedAt) { ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); ArgumentException.ThrowIfNullOrEmpty(sourceName); var advisoryKey = $"oracle/{dto.AdvisoryId}"; - var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime()); + var fetchProvenance = new AdvisoryProvenance(sourceName, "document", document.Uri, document.FetchedAt.ToUniversalTime()); + var mappingProvenance = new AdvisoryProvenance(sourceName, "mapping", dto.AdvisoryId, mappedAt.ToUniversalTime()); - var aliases = new List - { - $"ORACLE:{dto.AdvisoryId}", - }; - - var references = BuildReferences(dto, provenance).ToArray(); + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, sourceName, mappedAt); + var affectedPackages = BuildAffectedPackages(dto, sourceName, mappedAt); var advisory = new Advisory( advisoryKey, @@ -34,31 +46,88 @@ internal static class OracleMapper exploitKnown: false, aliases, references, - Array.Empty(), + affectedPackages, Array.Empty(), - new[] { provenance }); + new[] { fetchProvenance, mappingProvenance }); var flag = new PsirtFlagRecord( advisoryKey, "Oracle", sourceName, dto.AdvisoryId, - recordedAt.ToUniversalTime()); + mappedAt.ToUniversalTime()); return (advisory, flag); } - private static IEnumerable BuildReferences(OracleDto dto, AdvisoryProvenance provenance) + private static IReadOnlyList BuildAliases(OracleDto dto) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + $"ORACLE:{dto.AdvisoryId}".ToUpperInvariant(), + }; + + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve.Trim().ToUpperInvariant()); + } + } + + return aliases + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildReferences(OracleDto dto, string sourceName, DateTimeOffset recordedAt) { var comparer = StringComparer.OrdinalIgnoreCase; var entries = new List<(AdvisoryReference Reference, int Priority)> { - (new AdvisoryReference(dto.DetailUrl, "advisory", "oracle", null, provenance), 0), + (new AdvisoryReference( + dto.DetailUrl, + "advisory", + "oracle", + dto.Title, + new AdvisoryProvenance(sourceName, "reference", dto.DetailUrl, recordedAt.ToUniversalTime())), 0), }; + foreach (var document in dto.PatchDocuments) + { + var summary = document.Title ?? document.Product; + entries.Add((new AdvisoryReference( + document.Url, + "patch", + "oracle", + summary, + new AdvisoryProvenance(sourceName, "reference", document.Url, recordedAt.ToUniversalTime())), 1)); + } + foreach (var url in dto.References) { - entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1)); + entries.Add((new AdvisoryReference( + url, + "reference", + null, + null, + new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime())), 2)); + } + + foreach (var cve in dto.CveIds) + { + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + var cveUrl = $"https://www.cve.org/CVERecord?id={cve}"; + entries.Add((new AdvisoryReference( + cveUrl, + "advisory", + cve, + null, + new AdvisoryProvenance(sourceName, "reference", cveUrl, recordedAt.ToUniversalTime())), 3)); } return entries @@ -71,6 +140,287 @@ internal static class OracleMapper .OrderBy(t => t.Priority) .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) .ThenBy(t => t.Reference.Url, comparer) - .Select(t => t.Reference); + .Select(t => t.Reference) + .ToArray(); + } + + private static IReadOnlyList BuildAffectedPackages(OracleDto dto, string sourceName, DateTimeOffset recordedAt) + { + if (dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var entry in dto.Affected) + { + if (entry is null) + { + continue; + } + + var component = NormalizeComponent(entry.Component); + var notes = entry.Notes; + + foreach (var segment in SplitSupportedVersions(entry.Product, entry.SupportedVersions)) + { + if (string.IsNullOrWhiteSpace(segment.Product)) + { + continue; + } + + var identifier = CreateIdentifier(segment.Product, component); + var baseExpression = segment.Versions ?? entry.SupportedVersions ?? string.Empty; + var composedExpression = baseExpression; + + if (!string.IsNullOrEmpty(notes)) + { + composedExpression = string.IsNullOrEmpty(composedExpression) + ? $"notes: {notes}" + : $"{composedExpression} (notes: {notes})"; + } + + var rangeExpression = string.IsNullOrWhiteSpace(composedExpression) ? null : composedExpression; + var (fixedVersion, patchNumber) = ExtractFixMetadata(notes); + var rangeProvenance = new AdvisoryProvenance(sourceName, "range", identifier, recordedAt.ToUniversalTime()); + var rangePrimitives = BuildVendorRangePrimitives(entry, segment, component, baseExpression, rangeExpression, notes, fixedVersion, patchNumber); + + var ranges = rangeExpression is null && string.IsNullOrEmpty(fixedVersion) + ? Array.Empty() + : new[] + { + new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: fixedVersion, + lastAffectedVersion: null, + rangeExpression: rangeExpression, + provenance: rangeProvenance, + primitives: rangePrimitives), + }; + + var provenance = new[] + { + new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime()), + }; + + var package = new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + component, + ranges, + statuses: Array.Empty(), + provenance: provenance); + + var key = $"{identifier}::{component}::{ranges.FirstOrDefault()?.CreateDeterministicKey()}"; + if (seen.Add(key)) + { + packages.Add(package); + } + } + } + + return packages.Count == 0 ? Array.Empty() : packages; + } + + private static IEnumerable<(string Product, string? Versions)> SplitSupportedVersions(string product, string? supportedVersions) + { + var normalizedProduct = string.IsNullOrWhiteSpace(product) ? "Oracle Product" : product.Trim(); + + if (string.IsNullOrWhiteSpace(supportedVersions)) + { + yield return (normalizedProduct, null); + yield break; + } + + var segments = supportedVersions.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (segments.Length <= 1) + { + yield return (normalizedProduct, supportedVersions.Trim()); + yield break; + } + + foreach (var segment in segments) + { + var text = segment.Trim(); + if (text.Length == 0) + { + continue; + } + + var colonIndex = text.IndexOf(':'); + if (colonIndex > 0) + { + var name = text[..colonIndex].Trim(); + var versions = text[(colonIndex + 1)..].Trim(); + yield return (string.IsNullOrEmpty(name) ? normalizedProduct : name, versions); + } + else + { + yield return (normalizedProduct, text); + } + } + } + + private static RangePrimitives? BuildVendorRangePrimitives( + OracleAffectedEntry entry, + (string Product, string? Versions) segment, + string? component, + string? baseExpression, + string? rangeExpression, + string? notes, + string? fixedVersion, + string? patchNumber) + { + var extensions = new Dictionary(StringComparer.Ordinal); + + AddExtension(extensions, "oracle.product", segment.Product); + AddExtension(extensions, "oracle.productRaw", entry.Product); + AddExtension(extensions, "oracle.component", component); + AddExtension(extensions, "oracle.componentRaw", entry.Component); + AddExtension(extensions, "oracle.segmentVersions", segment.Versions); + AddExtension(extensions, "oracle.supportedVersions", entry.SupportedVersions); + AddExtension(extensions, "oracle.rangeExpression", rangeExpression); + AddExtension(extensions, "oracle.baseExpression", baseExpression); + AddExtension(extensions, "oracle.notes", notes); + AddExtension(extensions, "oracle.fixedVersion", fixedVersion); + AddExtension(extensions, "oracle.patchNumber", patchNumber); + + var versionTokens = ExtractVersionTokens(baseExpression); + if (versionTokens.Count > 0) + { + extensions["oracle.versionTokens"] = string.Join('|', versionTokens); + + var normalizedTokens = versionTokens + .Select(NormalizeSemVerToken) + .Where(static token => !string.IsNullOrEmpty(token)) + .Cast() + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (normalizedTokens.Length > 0) + { + extensions["oracle.versionTokens.normalized"] = string.Join('|', normalizedTokens); + } + } + + if (extensions.Count == 0) + { + return null; + } + + return new RangePrimitives(null, null, null, extensions); + } + + private static IReadOnlyList ExtractVersionTokens(string? baseExpression) + { + if (string.IsNullOrWhiteSpace(baseExpression)) + { + return Array.Empty(); + } + + var tokens = new List(); + foreach (var token in baseExpression.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) + { + var value = token.Trim(); + if (value.Length == 0 || !value.Any(char.IsDigit)) + { + continue; + } + + tokens.Add(value); + } + + return tokens.Count == 0 ? Array.Empty() : tokens; + } + + private static string? NormalizeSemVerToken(string token) + { + if (string.IsNullOrWhiteSpace(token)) + { + return null; + } + + if (PackageCoordinateHelper.TryParseSemVer(token, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized)) + { + return normalized; + } + + if (Version.TryParse(token, out var parsed)) + { + if (parsed.Build >= 0 && parsed.Revision >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}"; + } + + if (parsed.Build >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}"; + } + + return $"{parsed.Major}.{parsed.Minor}"; + } + + return null; + } + + private static void AddExtension(Dictionary extensions, string key, string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + extensions[key] = value.Trim(); + } + + private static string? NormalizeComponent(string? component) + { + if (string.IsNullOrWhiteSpace(component)) + { + return null; + } + + var trimmed = component.Trim(); + return trimmed.Length == 0 ? null : trimmed; + } + + private static string CreateIdentifier(string product, string? component) + { + var normalizedProduct = product.Trim(); + if (string.IsNullOrEmpty(component)) + { + return normalizedProduct; + } + + return $"{normalizedProduct}::{component}"; + } + + private static (string? FixedVersion, string? PatchNumber) ExtractFixMetadata(string? notes) + { + if (string.IsNullOrWhiteSpace(notes)) + { + return (null, null); + } + + string? fixedVersion = null; + string? patchNumber = null; + + var match = FixedVersionRegex.Match(notes); + if (match.Success) + { + fixedVersion = match.Groups["value"].Value.Trim(); + } + + match = PatchNumberRegex.Match(notes); + if (match.Success) + { + patchNumber = match.Groups["value"].Value.Trim(); + fixedVersion ??= patchNumber; + } + + return (fixedVersion, patchNumber); } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs index 4f766562..a9ffa7fb 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs @@ -1,7 +1,10 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text.RegularExpressions; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; @@ -10,22 +13,45 @@ internal static class OracleParser private static readonly Regex AnchorRegex = new("]+href=\"(?https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled); private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled); + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{3,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex UpdatedDateRegex = new("\"updatedDate\"\\s*:\\s*\"(?[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly string[] AllowedReferenceTokens = + { + "security-alerts", + "/kb/", + "/patches", + "/rs", + "/support/", + "/mos/", + "/technicalresources/", + "/technetwork/" + }; public static OracleDto Parse(string html, OracleDocumentMetadata metadata) { ArgumentException.ThrowIfNullOrEmpty(html); ArgumentNullException.ThrowIfNull(metadata); + var parser = new HtmlParser(); + var document = parser.ParseDocument(html); + + var published = ExtractPublishedDate(document) ?? metadata.Published; var content = Sanitize(html); + var affected = ExtractAffectedEntries(document); var references = ExtractReferences(html); + var patchDocuments = ExtractPatchDocuments(document, metadata.DetailUri); + var cveIds = ExtractCveIds(document, content, affected); return new OracleDto( metadata.AdvisoryId, metadata.Title, metadata.DetailUri.ToString(), - metadata.Published, + published, content, - references); + references, + cveIds, + affected, + patchDocuments); } private static string Sanitize(string html) @@ -40,14 +66,392 @@ internal static class OracleParser var references = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (Match match in AnchorRegex.Matches(html)) { - if (match.Success) + if (!match.Success) { - references.Add(match.Groups["url"].Value.Trim()); + continue; } + + var raw = match.Groups["url"].Value?.Trim(); + if (string.IsNullOrEmpty(raw)) + { + continue; + } + + var decoded = System.Net.WebUtility.HtmlDecode(raw) ?? raw; + + if (!Uri.TryCreate(decoded, UriKind.Absolute, out var uri)) + { + continue; + } + + if (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!ShouldIncludeReference(uri)) + { + continue; + } + + references.Add(uri.ToString()); } return references.Count == 0 ? Array.Empty() : references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray(); } + + private static bool ShouldIncludeReference(Uri uri) + { + if (uri.Host.EndsWith("cve.org", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (!uri.Host.EndsWith("oracle.com", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (uri.Query.Contains("type=doc", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + var path = uri.AbsolutePath ?? string.Empty; + return AllowedReferenceTokens.Any(token => path.Contains(token, StringComparison.OrdinalIgnoreCase)); + } + + private static DateTimeOffset? ExtractPublishedDate(IHtmlDocument document) + { + var meta = document.QuerySelectorAll("meta") + .FirstOrDefault(static element => string.Equals(element.GetAttribute("name"), "Updated Date", StringComparison.OrdinalIgnoreCase)); + if (meta is not null && TryParseOracleDate(meta.GetAttribute("content"), out var parsed)) + { + return parsed; + } + + foreach (var script in document.Scripts) + { + var text = script.TextContent; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + var match = UpdatedDateRegex.Match(text); + if (!match.Success) + { + continue; + } + + if (TryParseOracleDate(match.Groups["value"].Value, out var embedded)) + { + return embedded; + } + } + + return null; + } + + private static bool TryParseOracleDate(string? value, out DateTimeOffset result) + { + if (!string.IsNullOrWhiteSpace(value) + && DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out result)) + { + result = result.ToUniversalTime(); + return true; + } + + result = default; + return false; + } + + private static IReadOnlyList ExtractAffectedEntries(IHtmlDocument document) + { + var entries = new List(); + + foreach (var table in document.QuerySelectorAll("table")) + { + if (table is not IHtmlTableElement tableElement) + { + continue; + } + + if (!IsRiskMatrixTable(tableElement)) + { + continue; + } + + var lastProduct = string.Empty; + var lastComponent = string.Empty; + var lastVersions = string.Empty; + var lastNotes = string.Empty; + IReadOnlyList lastCves = Array.Empty(); + + foreach (var body in tableElement.Bodies) + { + foreach (var row in body.Rows) + { + if (row is not IHtmlTableRowElement tableRow || tableRow.Cells.Length == 0) + { + continue; + } + + var cveText = NormalizeCellText(GetCellText(tableRow, 0)); + var cves = ExtractCvesFromText(cveText); + if (cves.Count == 0 && lastCves.Count > 0) + { + cves = lastCves; + } + else if (cves.Count > 0) + { + lastCves = cves; + } + + var product = NormalizeCellText(GetCellText(tableRow, 1)); + if (string.IsNullOrEmpty(product)) + { + product = lastProduct; + } + else + { + lastProduct = product; + } + + var component = NormalizeCellText(GetCellText(tableRow, 2)); + if (string.IsNullOrEmpty(component)) + { + component = lastComponent; + } + else + { + lastComponent = component; + } + + var supportedVersions = NormalizeCellText(GetCellTextFromEnd(tableRow, 2)); + if (string.IsNullOrEmpty(supportedVersions)) + { + supportedVersions = lastVersions; + } + else + { + lastVersions = supportedVersions; + } + + var notes = NormalizeCellText(GetCellTextFromEnd(tableRow, 1)); + if (string.IsNullOrEmpty(notes)) + { + notes = lastNotes; + } + else + { + lastNotes = notes; + } + + if (string.IsNullOrEmpty(product) || cves.Count == 0) + { + continue; + } + + entries.Add(new OracleAffectedEntry( + product, + string.IsNullOrEmpty(component) ? null : component, + string.IsNullOrEmpty(supportedVersions) ? null : supportedVersions, + string.IsNullOrEmpty(notes) ? null : notes, + cves)); + } + } + } + + return entries.Count == 0 ? Array.Empty() : entries; + } + + private static IReadOnlyList ExtractCveIds(IHtmlDocument document, string content, IReadOnlyList affectedEntries) + { + var cves = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (!string.IsNullOrWhiteSpace(content)) + { + foreach (Match match in CveRegex.Matches(content)) + { + cves.Add(match.Value.ToUpperInvariant()); + } + } + + foreach (var entry in affectedEntries) + { + foreach (var cve in entry.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + cves.Add(cve.ToUpperInvariant()); + } + } + } + + var bodyText = document.Body?.TextContent; + if (!string.IsNullOrWhiteSpace(bodyText)) + { + foreach (Match match in CveRegex.Matches(bodyText)) + { + cves.Add(match.Value.ToUpperInvariant()); + } + } + + return cves.Count == 0 + ? Array.Empty() + : cves.OrderBy(static id => id, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static IReadOnlyList ExtractPatchDocuments(IHtmlDocument document, Uri detailUri) + { + var results = new List(); + + foreach (var table in document.QuerySelectorAll("table")) + { + if (table is not IHtmlTableElement tableElement) + { + continue; + } + + if (!TableHasPatchHeader(tableElement)) + { + continue; + } + + foreach (var body in tableElement.Bodies) + { + foreach (var row in body.Rows) + { + if (row is not IHtmlTableRowElement tableRow || tableRow.Cells.Length < 2) + { + continue; + } + + var product = NormalizeCellText(tableRow.Cells[0]?.TextContent); + if (string.IsNullOrEmpty(product)) + { + continue; + } + + var anchor = tableRow.Cells[1]?.QuerySelector("a"); + if (anchor is null) + { + continue; + } + + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + var decoded = System.Net.WebUtility.HtmlDecode(href) ?? href; + + if (!Uri.TryCreate(detailUri, decoded, out var uri) || !uri.IsAbsoluteUri) + { + continue; + } + + if (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var title = NormalizeCellText(anchor.TextContent); + results.Add(new OraclePatchDocument(product, string.IsNullOrEmpty(title) ? null : title, uri.ToString())); + } + } + } + + return results.Count == 0 ? Array.Empty() : results; + } + + private static bool IsRiskMatrixTable(IHtmlTableElement table) + { + var headerText = table.Head?.TextContent; + if (string.IsNullOrWhiteSpace(headerText)) + { + return false; + } + + return headerText.Contains("CVE ID", StringComparison.OrdinalIgnoreCase) + && headerText.Contains("Supported Versions", StringComparison.OrdinalIgnoreCase); + } + + private static bool TableHasPatchHeader(IHtmlTableElement table) + { + var headerText = table.Head?.TextContent; + if (string.IsNullOrWhiteSpace(headerText)) + { + return false; + } + + return headerText.Contains("Affected Products and Versions", StringComparison.OrdinalIgnoreCase) + && headerText.Contains("Patch Availability Document", StringComparison.OrdinalIgnoreCase); + } + + private static string? GetCellText(IHtmlTableRowElement row, int index) + { + if (index < 0 || index >= row.Cells.Length) + { + return null; + } + + return row.Cells[index]?.TextContent; + } + + private static string? GetCellTextFromEnd(IHtmlTableRowElement row, int offsetFromEnd) + { + if (offsetFromEnd <= 0) + { + return null; + } + + var index = row.Cells.Length - offsetFromEnd; + return index >= 0 ? row.Cells[index]?.TextContent : null; + } + + private static IReadOnlyList ExtractCvesFromText(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return Array.Empty(); + } + + var matches = CveRegex.Matches(text); + if (matches.Count == 0) + { + return Array.Empty(); + } + + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in matches) + { + if (match.Success) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + + return set.Count == 0 + ? Array.Empty() + : set.OrderBy(static id => id, StringComparer.Ordinal).ToArray(); + } + + private static string NormalizeCellText(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var cleaned = value.Replace('\u00A0', ' '); + cleaned = WhitespaceRegex.Replace(cleaned, " "); + return cleaned.Trim(); + } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs new file mode 100644 index 00000000..3fca3f5f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OraclePatchDocument( + [property: JsonPropertyName("product")] string Product, + [property: JsonPropertyName("title")] string? Title, + [property: JsonPropertyName("url")] string Url); diff --git a/src/Jobs.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs similarity index 100% rename from src/Jobs.cs rename to src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs index 2edd7b0c..9ccdef8f 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs @@ -35,6 +35,7 @@ public sealed class OracleConnector : IFeedConnector private readonly IAdvisoryStore _advisoryStore; private readonly IPsirtFlagStore _psirtFlagStore; private readonly ISourceStateRepository _stateRepository; + private readonly OracleCalendarFetcher _calendarFetcher; private readonly OracleOptions _options; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; @@ -47,6 +48,7 @@ public sealed class OracleConnector : IFeedConnector IAdvisoryStore advisoryStore, IPsirtFlagStore psirtFlagStore, ISourceStateRepository stateRepository, + OracleCalendarFetcher calendarFetcher, IOptions options, TimeProvider? timeProvider, ILogger logger) @@ -58,6 +60,7 @@ public sealed class OracleConnector : IFeedConnector _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _calendarFetcher = calendarFetcher ?? throw new ArgumentNullException(nameof(calendarFetcher)); _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); _options.Validate(); _timeProvider = timeProvider ?? TimeProvider.System; @@ -71,14 +74,21 @@ public sealed class OracleConnector : IFeedConnector var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var pendingDocuments = cursor.PendingDocuments.ToList(); var pendingMappings = cursor.PendingMappings.ToList(); + var fetchCache = new Dictionary(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); var now = _timeProvider.GetUtcNow(); - foreach (var uri in _options.AdvisoryUris) + var advisoryUris = await ResolveAdvisoryUrisAsync(cancellationToken).ConfigureAwait(false); + + foreach (var uri in advisoryUris) { cancellationToken.ThrowIfCancellationRequested(); try { + var cacheKey = uri.AbsoluteUri; + touchedResources.Add(cacheKey); + var advisoryId = DeriveAdvisoryId(uri); var title = advisoryId.Replace('-', ' '); var published = now; @@ -100,6 +110,22 @@ public sealed class OracleConnector : IFeedConnector continue; } + var cacheEntry = OracleFetchCacheEntry.FromDocument(result.Document); + if (existing is not null + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal) + && cursor.TryGetFetchCache(cacheKey, out var cached) + && cached.Matches(result.Document)) + { + _logger.LogDebug("Oracle advisory {AdvisoryId} unchanged; skipping parse/map", advisoryId); + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + fetchCache[cacheKey] = cacheEntry; + continue; + } + + fetchCache[cacheKey] = cacheEntry; + if (!pendingDocuments.Contains(result.Document.Id)) { pendingDocuments.Add(result.Document.Id); @@ -118,9 +144,19 @@ public sealed class OracleConnector : IFeedConnector } } + if (fetchCache.Count > 0 && touchedResources.Count > 0) + { + var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in stale) + { + fetchCache.Remove(key); + } + } + var updatedCursor = cursor .WithPendingDocuments(pendingDocuments) .WithPendingMappings(pendingMappings) + .WithFetchCache(fetchCache) .WithLastProcessed(now); await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); @@ -175,6 +211,17 @@ public sealed class OracleConnector : IFeedConnector continue; } + if (!OracleDtoValidator.TryNormalize(dto, out var normalized, out var validationError)) + { + _logger.LogWarning("Oracle validation failed for document {DocumentId}: {Reason}", document.Id, validationError ?? "unknown"); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + dto = normalized; + var json = JsonSerializer.Serialize(dto, SerializerOptions); var payload = BsonDocument.Parse(json); var validatedAt = _timeProvider.GetUtcNow(); @@ -252,7 +299,7 @@ public sealed class OracleConnector : IFeedConnector } var mappedAt = _timeProvider.GetUtcNow(); - var (advisory, flag) = OracleMapper.Map(dto, SourceName, mappedAt); + var (advisory, flag) = OracleMapper.Map(dto, document, dtoRecord, SourceName, mappedAt); await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); @@ -276,6 +323,30 @@ public sealed class OracleConnector : IFeedConnector await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); } + private async Task> ResolveAdvisoryUrisAsync(CancellationToken cancellationToken) + { + var uris = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var uri in _options.AdvisoryUris) + { + if (uri is not null) + { + uris.Add(uri.AbsoluteUri); + } + } + + var calendarUris = await _calendarFetcher.GetAdvisoryUrisAsync(cancellationToken).ConfigureAwait(false); + foreach (var uri in calendarUris) + { + uris.Add(uri.AbsoluteUri); + } + + return uris + .Select(static value => new Uri(value, UriKind.Absolute)) + .OrderBy(static value => value.AbsoluteUri, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + private static string DeriveAdvisoryId(Uri uri) { var segments = uri.Segments; diff --git a/src/OracleDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs similarity index 100% rename from src/OracleDependencyInjectionRoutine.cs rename to src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs index 1c7ce3fd..03a4cd61 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs @@ -29,8 +29,13 @@ public static class OracleServiceCollectionExtensions { clientOptions.AllowedHosts.Add(uri.Host); } + foreach (var uri in options.CalendarUris) + { + clientOptions.AllowedHosts.Add(uri.Host); + } }); + services.AddTransient(); services.AddTransient(); return services; } diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..e0b1abdd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Oracle.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md index 2fb4ddb4..020362b9 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md @@ -1,12 +1,13 @@ # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|Oracle options & HttpClient configuration|BE-Conn-Oracle|Source.Common|TODO – add options with base URLs and allowlisted HttpClient configuration.| -|CPU calendar plus advisory fetchers|BE-Conn-Oracle|Source.Common|TODO – implement CPU calendar scraper and detail fetch pipeline.| -|Extractor for products/components/fix levels|BE-Conn-Oracle|Source.Common|TODO – normalize product/component names with fixed patch info.| -|DTO schema and validation|BE-Conn-Oracle, QA|Source.Common|TODO – define DTO + validation to quarantine malformed rows.| -|Canonical mapping with psirt_flags|BE-Conn-Oracle|Models|TODO – map advisories with psirt flags and vendor metadata.| -|SourceState and dedupe|BE-Conn-Oracle|Storage.Mongo|TODO – persist cursor/backoff and hash-based dedupe.| -|Golden fixtures and precedence tests (later with merge)|QA|Source.Vndr.Oracle|TODO – add fixtures verifying PSIRT overrides NVD ranges.| -|Dependency injection routine & job registration|BE-Conn-Oracle|Core|TODO – add DI routine plus scheduler registrations similar to other connectors.| -|Implement Oracle connector skeleton|BE-Conn-Oracle|Source.Common|TODO – replace stub plugin with full pipeline for fetching, DTO storage, and mapping.| +|Oracle options & HttpClient configuration|BE-Conn-Oracle|Source.Common|**DONE** – `AddOracleConnector` wires options and allowlisted HttpClient.| +|CPU calendar plus advisory fetchers|BE-Conn-Oracle|Source.Common|**DONE** – resume/backfill scenario covered with new integration test and fetch cache pruning verified.| +|Extractor for products/components/fix levels|BE-Conn-Oracle|Source.Common|**DONE** – HTML risk matrices parsed into vendor packages with fix heuristics and normalized versions.| +|DTO schema and validation|BE-Conn-Oracle, QA|Source.Common|**DONE** – `OracleDtoValidator` enforces required fields and quarantines malformed payloads.| +|Canonical mapping with psirt_flags|BE-Conn-Oracle|Models|**DONE** – mapper now emits CVE aliases, patch references, and vendor affected packages under psirt flag provenance.| +|SourceState and dedupe|BE-Conn-Oracle|Storage.Mongo|**DONE** – cursor fetch cache tracks SHA/ETag to skip unchanged advisories and clear pending work.| +|Golden fixtures and precedence tests (later with merge)|QA|Source.Vndr.Oracle|**DONE** – snapshot fixtures and psirt flag assertions added in `OracleConnectorTests`.| +|Dependency injection routine & job registration|BE-Conn-Oracle|Core|**DONE** – `OracleDependencyInjectionRoutine` registers connector and fetch/parse/map jobs with scheduler defaults.| +|Implement Oracle connector skeleton|BE-Conn-Oracle|Source.Common|**DONE** – fetch/parse/map pipeline persists documents, DTOs, advisories, psirt flags.| +|Range primitives & provenance backfill|BE-Conn-Oracle|Models, Storage.Mongo|**DONE** – vendor primitives emitted (extensions + fix parsing), provenance tagging/logging extended, snapshots refreshed.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj index 3f55e9bc..0f345b83 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj @@ -10,4 +10,9 @@ + + + PreserveNewest + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json new file mode 100644 index 00000000..1636db38 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json @@ -0,0 +1,258 @@ +[ + { + "advisoryKey": "VMSA-2024-0001", + "affectedPackages": [ + { + "identifier": "VMware ESXi 7.0", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware ESXi 7.0" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "7.0u3f", + "introducedVersion": "7.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": null, + "fixedInclusive": false, + "introduced": "7.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware ESXi 7.0", + "vmware.version.raw": "7.0", + "vmware.fixedVersion.raw": "7.0u3f" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware ESXi 7.0" + }, + "rangeExpression": "7.0", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "VMware vCenter Server 8.0", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware vCenter Server 8.0" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "8.0a", + "introducedVersion": "8.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": null, + "fixedInclusive": false, + "introduced": "8.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware vCenter Server 8.0", + "vmware.version.raw": "8.0", + "vmware.fixedVersion.raw": "8.0a" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware vCenter Server 8.0" + }, + "rangeExpression": "8.0", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-1000", + "CVE-2024-1001", + "VMSA-2024-0001" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-01T10:00:00+00:00", + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://vmware.example/api/vmsa/VMSA-2024-0001.json" + }, + { + "kind": "mapping", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMSA-2024-0001" + } + ], + "published": "2024-04-01T10:00:00+00:00", + "references": [ + { + "kind": "kb", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://kb.vmware.example/90234" + }, + "sourceTag": "kb", + "summary": null, + "url": "https://kb.vmware.example/90234" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" + }, + "sourceTag": "advisory", + "summary": null, + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" + } + ], + "severity": null, + "summary": "Security updates for VMware ESXi 7.0 and vCenter Server 8.0 resolve multiple vulnerabilities.", + "title": "VMware ESXi and vCenter Server updates address vulnerabilities" + }, + { + "advisoryKey": "VMSA-2024-0002", + "affectedPackages": [ + { + "identifier": "VMware Cloud Foundation 5.x", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware Cloud Foundation 5.x" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "5.1.1", + "introducedVersion": "5.1", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "5.1.1", + "fixedInclusive": false, + "introduced": "5.1", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware Cloud Foundation 5.x", + "vmware.version.raw": "5.1", + "vmware.fixedVersion.raw": "5.1.1" + } + }, + "provenance": { + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware Cloud Foundation 5.x" + }, + "rangeExpression": "5.1", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-2000", + "VMSA-2024-0002" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-02T09:00:00+00:00", + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://vmware.example/api/vmsa/VMSA-2024-0002.json" + }, + { + "kind": "mapping", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMSA-2024-0002" + } + ], + "published": "2024-04-02T09:00:00+00:00", + "references": [ + { + "kind": "kb", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://kb.vmware.example/91234" + }, + "sourceTag": "kb", + "summary": null, + "url": "https://kb.vmware.example/91234" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" + }, + "sourceTag": "advisory", + "summary": null, + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" + } + ], + "severity": null, + "summary": "An update is available for VMware Cloud Foundation components to address a remote code execution vulnerability.", + "title": "VMware Cloud Foundation remote code execution vulnerability" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json new file mode 100644 index 00000000..bc265ee5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json @@ -0,0 +1,33 @@ +{ + "id": "VMSA-2024-0001", + "title": "VMware ESXi and vCenter Server updates address vulnerabilities", + "summary": "Security updates for VMware ESXi 7.0 and vCenter Server 8.0 resolve multiple vulnerabilities.", + "published": "2024-04-01T10:00:00Z", + "modified": "2024-04-01T10:00:00Z", + "cves": [ + "CVE-2024-1000", + "CVE-2024-1001" + ], + "affected": [ + { + "product": "VMware ESXi 7.0", + "version": "7.0", + "fixedVersion": "7.0u3f" + }, + { + "product": "VMware vCenter Server 8.0", + "version": "8.0", + "fixedVersion": "8.0a" + } + ], + "references": [ + { + "type": "kb", + "url": "https://kb.vmware.example/90234" + }, + { + "type": "advisory", + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json new file mode 100644 index 00000000..a78af8ce --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json @@ -0,0 +1,27 @@ +{ + "id": "VMSA-2024-0002", + "title": "VMware Cloud Foundation remote code execution vulnerability", + "summary": "An update is available for VMware Cloud Foundation components to address a remote code execution vulnerability.", + "published": "2024-04-02T09:00:00Z", + "modified": "2024-04-02T09:00:00Z", + "cves": [ + "CVE-2024-2000" + ], + "affected": [ + { + "product": "VMware Cloud Foundation 5.x", + "version": "5.1", + "fixedVersion": "5.1.1" + } + ], + "references": [ + { + "type": "kb", + "url": "https://kb.vmware.example/91234" + }, + { + "type": "advisory", + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json new file mode 100644 index 00000000..979f8e2e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json @@ -0,0 +1,23 @@ +{ + "id": "VMSA-2024-0003", + "title": "VMware NSX-T advisory for input validation issue", + "summary": "VMware NSX-T has resolved an input validation vulnerability impacting API endpoints.", + "published": "2024-04-03T08:15:00Z", + "modified": "2024-04-03T08:15:00Z", + "cves": [ + "CVE-2024-3000" + ], + "affected": [ + { + "product": "VMware NSX-T 3.2", + "version": "3.2", + "fixedVersion": "3.2.3" + } + ], + "references": [ + { + "type": "kb", + "url": "https://kb.vmware.example/93456" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json new file mode 100644 index 00000000..fff55129 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json @@ -0,0 +1,12 @@ +[ + { + "id": "VMSA-2024-0001", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0001.json", + "modified": "2024-04-01T10:00:00Z" + }, + { + "id": "VMSA-2024-0002", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0002.json", + "modified": "2024-04-02T09:00:00Z" + } +] diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json new file mode 100644 index 00000000..86ca0074 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json @@ -0,0 +1,17 @@ +[ + { + "id": "VMSA-2024-0001", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0001.json", + "modified": "2024-04-01T10:00:00Z" + }, + { + "id": "VMSA-2024-0002", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0002.json", + "modified": "2024-04-02T09:00:00Z" + }, + { + "id": "VMSA-2024-0003", + "url": "https://vmware.example/api/vmsa/VMSA-2024-0003.json", + "modified": "2024-04-03T08:15:00Z" + } +] diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs new file mode 100644 index 00000000..6205baff --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Vndr.Vmware; +using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; +using StellaOps.Feedser.Source.Vndr.Vmware.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Tests.Vmware; + +[Collection("mongo-fixture")] +public sealed class VmwareConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private readonly ITestOutputHelper _output; + + private static readonly Uri IndexUri = new("https://vmware.example/api/vmsa/index.json"); + private static readonly Uri DetailOne = new("https://vmware.example/api/vmsa/VMSA-2024-0001.json"); + private static readonly Uri DetailTwo = new("https://vmware.example/api/vmsa/VMSA-2024-0002.json"); + private static readonly Uri DetailThree = new("https://vmware.example/api/vmsa/VMSA-2024-0003.json"); + + public VmwareConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 5, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + _output = output; + } + + [Fact] + public async Task FetchParseMap_ProducesSnapshotAndCoversResume() + { + await using var provider = await BuildServiceProviderAsync(); + SeedInitialResponses(); + + using var metrics = new VmwareMetricCollector(); + + var connector = provider.GetRequiredService(); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); + + var snapshot = Normalize(SnapshotSerializer.ToSnapshot(ordered)); + var expected = Normalize(ReadFixture("vmware-advisories.snapshot.json")); + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", "vmware-advisories.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + + var psirtCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + var psirtFlags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); + _output.WriteLine("PSIRT flags after initial map: " + string.Join(", ", psirtFlags.Select(flag => flag.GetValue("_id", BsonValue.Create("")).ToString()))); + Assert.Equal(2, psirtFlags.Count); + Assert.All(psirtFlags, doc => Assert.Equal("VMware", doc["vendor"].AsString)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VmwareConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Empty(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) ? pendingDocs.AsBsonArray : new BsonArray()); + Assert.Empty(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) ? pendingMaps.AsBsonArray : new BsonArray()); + var cursorSnapshot = VmwareCursor.FromBson(state.Cursor); + _output.WriteLine($"Initial fetch cache entries: {cursorSnapshot.FetchCache.Count}"); + foreach (var entry in cursorSnapshot.FetchCache) + { + _output.WriteLine($"Cache seed: {entry.Key} -> {entry.Value.Sha256}"); + } + + // Second run with unchanged advisories and one new advisory. + SeedUpdateResponses(); + _timeProvider.Advance(TimeSpan.FromHours(1)); + + await connector.FetchAsync(provider, CancellationToken.None); + var documentStore = provider.GetRequiredService(); + var resumeDocOne = await documentStore.FindBySourceAndUriAsync(VmwareConnectorPlugin.SourceName, DetailOne.ToString(), CancellationToken.None); + var resumeDocTwo = await documentStore.FindBySourceAndUriAsync(VmwareConnectorPlugin.SourceName, DetailTwo.ToString(), CancellationToken.None); + _output.WriteLine($"After resume fetch status: {resumeDocOne?.Status} ({resumeDocOne?.Sha256}), {resumeDocTwo?.Status} ({resumeDocTwo?.Sha256})"); + Assert.Equal(DocumentStatuses.Mapped, resumeDocOne?.Status); + Assert.Equal(DocumentStatuses.Mapped, resumeDocTwo?.Status); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(3, advisories.Count); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "VMSA-2024-0003"); + + psirtFlags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); + _output.WriteLine("PSIRT flags after resume: " + string.Join(", ", psirtFlags.Select(flag => flag.GetValue("_id", BsonValue.Create("")).ToString()))); + Assert.Equal(3, psirtFlags.Count); + Assert.Contains(psirtFlags, doc => doc["_id"] == "VMSA-2024-0003"); + + var measurements = metrics.Measurements; + _output.WriteLine("Captured metrics:"); + foreach (var measurement in measurements) + { + _output.WriteLine($"{measurement.Name} -> {measurement.Value}"); + } + + Assert.Equal(0, Sum(measurements, "vmware.fetch.failures")); + Assert.Equal(0, Sum(measurements, "vmware.parse.fail")); + Assert.Equal(3, Sum(measurements, "vmware.fetch.items")); // two initial, one new + + var affectedCounts = measurements + .Where(m => m.Name == "vmware.map.affected_count") + .Select(m => (int)m.Value) + .OrderBy(v => v) + .ToArray(); + Assert.Equal(new[] { 1, 1, 2 }, affectedCounts); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _handler.Clear(); + return Task.CompletedTask; + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddVmwareConnector(opts => + { + opts.IndexUri = IndexUri; + opts.InitialBackfill = TimeSpan.FromDays(30); + opts.ModifiedTolerance = TimeSpan.FromMinutes(5); + opts.MaxAdvisoriesPerFetch = 10; + opts.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(VmwareOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + _handler.AddJsonResponse(IndexUri, ReadFixture("vmware-index-initial.json")); + _handler.AddJsonResponse(DetailOne, ReadFixture("vmware-detail-vmsa-2024-0001.json")); + _handler.AddJsonResponse(DetailTwo, ReadFixture("vmware-detail-vmsa-2024-0002.json")); + } + + private void SeedUpdateResponses() + { + _handler.AddJsonResponse(IndexUri, ReadFixture("vmware-index-second.json")); + _handler.AddJsonResponse(DetailOne, ReadFixture("vmware-detail-vmsa-2024-0001.json")); + _handler.AddJsonResponse(DetailTwo, ReadFixture("vmware-detail-vmsa-2024-0002.json")); + _handler.AddJsonResponse(DetailThree, ReadFixture("vmware-detail-vmsa-2024-0003.json")); + } + + private static string ReadFixture(string name) + { + var primary = Path.Combine(AppContext.BaseDirectory, "Vmware", "Fixtures", name); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(AppContext.BaseDirectory, "Fixtures", name); + if (File.Exists(fallback)) + { + return File.ReadAllText(fallback); + } + + throw new FileNotFoundException($"Fixture '{name}' not found.", name); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd(); + + private static long Sum(IEnumerable measurements, string name) + => measurements.Where(m => m.Name == name).Sum(m => m.Value); + + private sealed class VmwareMetricCollector : IDisposable + { + private readonly MeterListener _listener; + private readonly ConcurrentBag _measurements = new(); + + public VmwareMetricCollector() + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == VmwareDiagnostics.MeterName) + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + + _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var tagList = new List>(tags.Length); + foreach (var tag in tags) + { + tagList.Add(tag); + } + + _measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagList)); + }); + + _listener.Start(); + } + + public IReadOnlyCollection Measurements => _measurements; + + public void Dispose() => _listener.Dispose(); + + public sealed record MetricMeasurement(string Name, long Value, IReadOnlyList> Tags); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs index b88d72fc..354655d9 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs @@ -67,7 +67,7 @@ public sealed class VmwareMapperTests var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, VmwareConnectorPlugin.SourceName, "vmware.v1", payload, DateTimeOffset.UtcNow); - var advisory = VmwareMapper.Map(dto, document, dtoRecord); + var (advisory, flag) = VmwareMapper.Map(dto, document, dtoRecord); Assert.Equal(dto.AdvisoryId, advisory.AdvisoryKey); Assert.Contains("CVE-2025-0001", advisory.Aliases); @@ -79,5 +79,8 @@ public sealed class VmwareMapperTests Assert.Equal("7.0u3", advisory.AffectedPackages[0].VersionRanges[0].FixedVersion); Assert.Equal(2, advisory.References.Length); Assert.Equal("https://kb.vmware.com/some-kb", advisory.References[0].Url); + Assert.Equal(dto.AdvisoryId, flag.AdvisoryKey); + Assert.Equal("VMware", flag.Vendor); + Assert.Equal(VmwareConnectorPlugin.SourceName, flag.SourceName); } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md index 5f1b714e..18c7e518 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md @@ -20,10 +20,9 @@ VMware/Broadcom PSIRT connector ingesting VMSA advisories; authoritative for VMw In: PSIRT precedence mapping, affected/fixedBy extraction, advisory references. Out: customer portal authentication flows beyond public advisories; downloading patches. ## Observability & security expectations -- Metrics: vmware.fetch.items, vmware.parse.fail, vmware.map.affected_count. +- Metrics: SourceDiagnostics emits shared `feedser.source.http.*` counters/histograms tagged `feedser.source=vmware`, allowing dashboards to measure fetch volume, parse failures, and map affected counts without bespoke metric names. - Logs: vmsa ids, product counts, extraction timings; handle portal rate limits politely. ## Tests - Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Vmware.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs index 08791f8c..c2b75f71 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs @@ -9,12 +9,15 @@ internal sealed record VmwareCursor( DateTimeOffset? LastModified, IReadOnlyCollection ProcessedIds, IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings) + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) { private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); private static readonly IReadOnlyCollection EmptyStringList = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyFetchCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); - public static VmwareCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList); + public static VmwareCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyFetchCache); public BsonDocument ToBsonDocument() { @@ -34,6 +37,17 @@ internal sealed record VmwareCursor( document["processedIds"] = new BsonArray(ProcessedIds); } + if (FetchCache.Count > 0) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBsonDocument(); + } + + document["fetchCache"] = cacheDocument; + } + return document; } @@ -49,13 +63,17 @@ internal sealed record VmwareCursor( : null; var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray idsArray - ? idsArray.OfType().Where(static x => x.BsonType == BsonType.String).Select(static x => x.AsString).ToArray() + ? idsArray.OfType() + .Where(static x => x.BsonType == BsonType.String) + .Select(static x => x.AsString) + .ToArray() : EmptyStringList; var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); - return new VmwareCursor(lastModified, processedIds, pendingDocuments, pendingMappings); + return new VmwareCursor(lastModified, processedIds, pendingDocuments, pendingMappings, fetchCache); } public VmwareCursor WithLastModified(DateTimeOffset timestamp, IEnumerable processedIds) @@ -74,6 +92,27 @@ internal sealed record VmwareCursor( public VmwareCursor WithPendingMappings(IEnumerable ids) => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + public VmwareCursor WithFetchCache(IDictionary? cache) + { + if (cache is null || cache.Count == 0) + { + return this with { FetchCache = EmptyFetchCache }; + } + + return this with { FetchCache = new Dictionary(cache, StringComparer.OrdinalIgnoreCase) }; + } + + public bool TryGetFetchCache(string key, out VmwareFetchCacheEntry entry) + { + if (FetchCache.Count == 0) + { + entry = VmwareFetchCacheEntry.Empty; + return false; + } + + return FetchCache.TryGetValue(key, out entry!); + } + public VmwareCursor AddProcessedId(string id) { if (string.IsNullOrWhiteSpace(id)) @@ -104,13 +143,30 @@ internal sealed record VmwareCursor( return results; } - private static DateTimeOffset? ParseDate(BsonValue value) + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) { - return value.BsonType switch + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument || cacheDocument.ElementCount == 0) + { + return EmptyFetchCache; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entryDocument) + { + cache[element.Name] = VmwareFetchCacheEntry.FromBson(entryDocument); + } + } + + return cache; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch { BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), _ => null, }; - } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs new file mode 100644 index 00000000..1bfd980c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs @@ -0,0 +1,88 @@ +using System; +using MongoDB.Bson; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal sealed record VmwareFetchCacheEntry(string? Sha256, string? ETag, DateTimeOffset? LastModified) +{ + public static VmwareFetchCacheEntry Empty { get; } = new(string.Empty, null, null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["sha256"] = Sha256 ?? string.Empty, + }; + + if (!string.IsNullOrWhiteSpace(ETag)) + { + document["etag"] = ETag; + } + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + return document; + } + + public static VmwareFetchCacheEntry FromBson(BsonDocument document) + { + var sha256 = document.TryGetValue("sha256", out var shaValue) ? shaValue.ToString() : string.Empty; + string? etag = null; + if (document.TryGetValue("etag", out var etagValue) && !etagValue.IsBsonNull) + { + etag = etagValue.ToString(); + } + + DateTimeOffset? lastModified = null; + if (document.TryGetValue("lastModified", out var lastModifiedValue)) + { + lastModified = lastModifiedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new VmwareFetchCacheEntry(sha256, etag, lastModified); + } + + public static VmwareFetchCacheEntry FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + return new VmwareFetchCacheEntry( + document.Sha256, + document.Etag, + document.LastModified?.ToUniversalTime()); + } + + public bool Matches(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + if (!string.IsNullOrEmpty(Sha256) && !string.IsNullOrEmpty(document.Sha256) + && string.Equals(Sha256, document.Sha256, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (!string.IsNullOrEmpty(ETag) && !string.IsNullOrEmpty(document.Etag) + && string.Equals(ETag, document.Etag, StringComparison.Ordinal)) + { + return true; + } + + if (LastModified.HasValue && document.LastModified.HasValue + && LastModified.Value.ToUniversalTime() == document.LastModified.Value.ToUniversalTime()) + { + return true; + } + + return false; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs index 4084c6fb..91161c8e 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs @@ -3,27 +3,30 @@ using System.Collections.Generic; using System.Linq; using StellaOps.Feedser.Models; using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Packages; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; internal static class VmwareMapper { - public static Advisory Map(VmwareDetailDto dto, DocumentRecord document, DtoRecord dtoRecord) + public static (Advisory Advisory, PsirtFlagRecord Flag) Map(VmwareDetailDto dto, DocumentRecord document, DtoRecord dtoRecord) { ArgumentNullException.ThrowIfNull(dto); ArgumentNullException.ThrowIfNull(document); ArgumentNullException.ThrowIfNull(dtoRecord); - var fetchProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt); - var mappingProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "mapping", dto.AdvisoryId, dtoRecord.ValidatedAt); + var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); + var fetchProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt.ToUniversalTime()); + var mappingProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "mapping", dto.AdvisoryId, recordedAt); var aliases = BuildAliases(dto); - var references = BuildReferences(dto, dtoRecord.ValidatedAt); - var affectedPackages = BuildAffectedPackages(dto, dtoRecord.ValidatedAt); + var references = BuildReferences(dto, recordedAt); + var affectedPackages = BuildAffectedPackages(dto, recordedAt); - return new Advisory( + var advisory = new Advisory( dto.AdvisoryId, dto.Title, dto.Summary, @@ -37,6 +40,15 @@ internal static class VmwareMapper affectedPackages, cvssMetrics: Array.Empty(), provenance: new[] { fetchProvenance, mappingProvenance }); + + var flag = new PsirtFlagRecord( + dto.AdvisoryId, + "VMware", + VmwareConnectorPlugin.SourceName, + dto.AdvisoryId, + recordedAt); + + return (advisory, flag); } private static IEnumerable BuildAliases(VmwareDetailDto dto) @@ -127,13 +139,15 @@ internal static class VmwareMapper var ranges = new List(); if (!string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.FixedVersion)) { + var rangeProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "range", product.Product, recordedAt); ranges.Add(new AffectedVersionRange( rangeKind: "vendor", introducedVersion: product.Version, fixedVersion: product.FixedVersion, lastAffectedVersion: null, rangeExpression: product.Version, - provenance: new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "range", product.Product, recordedAt))); + provenance: rangeProvenance, + primitives: BuildRangePrimitives(product))); } packages.Add(new AffectedPackage( @@ -147,4 +161,75 @@ internal static class VmwareMapper return packages; } + + private static RangePrimitives? BuildRangePrimitives(VmwareAffectedProductDto product) + { + var extensions = new Dictionary(StringComparer.Ordinal); + AddExtension(extensions, "vmware.product", product.Product); + AddExtension(extensions, "vmware.version.raw", product.Version); + AddExtension(extensions, "vmware.fixedVersion.raw", product.FixedVersion); + + var semVer = BuildSemVerPrimitive(product.Version, product.FixedVersion); + if (semVer is null && extensions.Count == 0) + { + return null; + } + + return new RangePrimitives(semVer, null, null, extensions.Count == 0 ? null : extensions); + } + + private static SemVerPrimitive? BuildSemVerPrimitive(string? introduced, string? fixedVersion) + { + var introducedNormalized = NormalizeSemVer(introduced); + var fixedNormalized = NormalizeSemVer(fixedVersion); + + if (introducedNormalized is null && fixedNormalized is null) + { + return null; + } + + return new SemVerPrimitive( + introducedNormalized, + IntroducedInclusive: true, + fixedNormalized, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: null); + } + + private static string? NormalizeSemVer(string? value) + { + if (PackageCoordinateHelper.TryParseSemVer(value, out _, out var normalized) && !string.IsNullOrWhiteSpace(normalized)) + { + return normalized; + } + + if (Version.TryParse(value, out var parsed)) + { + if (parsed.Build >= 0 && parsed.Revision >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}.{parsed.Revision}"; + } + + if (parsed.Build >= 0) + { + return $"{parsed.Major}.{parsed.Minor}.{parsed.Build}"; + } + + return $"{parsed.Major}.{parsed.Minor}"; + } + + return null; + } + + private static void AddExtension(Dictionary extensions, string key, string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + extensions[key] = value.Trim(); + } } diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..aee480af --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Vmware.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md index e35ff49f..cf7ba462 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md @@ -2,15 +2,16 @@ | ID | Task | Owner | Status | Depends On | Notes | |------|-----------------------------------------------|-------|--------|------------|-------| -| VM1 | Advisory listing discovery + cursor | Conn | TODO | Common | Track revisions; respect VMware PSIRT cadence. | -| VM2 | VMSA parser → DTO | QA | TODO | | Extract product/version/CVE/severity; capture fixed build numbers. | -| VM3 | Canonical mapping (aliases/affected/refs) | Conn | TODO | Models | Deterministic ordering; set vendor="VMware" and advisory_id_text=VMSA. | -| VM4 | Snapshot tests + resume | QA | TODO | Storage | | -| VM5 | Observability | QA | TODO | | Metrics counters. | -| VM6 | SourceState + hash dedupe | Conn | TODO | Storage | Skip unchanged advisories; ensure idempotent reruns. | -| VM6a | Options & HttpClient configuration | Conn | TODO | Source.Common | Introduce `VmwareOptions` with base portal URLs and allowlisted HttpClient setup. | -| VM7 | Dependency injection routine & scheduler registration | Conn | TODO | Core | Wire HttpClient/options and register fetch/parse/map jobs consistent with other connectors. | -| VM8 | Replace stub plugin with connector pipeline skeleton | Conn | TODO | Source.Common | Implement fetch/parse/map scaffolding persisting source_state, documents, and canonical advisories. | +| VM1 | Advisory listing discovery + cursor | Conn | DONE | Common | **DONE** – fetch pipeline uses index JSON with sliding cursor + processed id tracking. | +| VM2 | VMSA parser → DTO | QA | DONE | | **DONE** – JSON DTO deserialization wired with sanitization. | +| VM3 | Canonical mapping (aliases/affected/refs) | Conn | DONE | Models | **DONE** – `VmwareMapper` emits aliases/affected/reference ordering and persists PSIRT flags via `PsirtFlagStore`. | +| VM4 | Snapshot tests + resume | QA | DONE | Storage | **DONE** – integration test validates snapshot output and resume flow with cached state. | +| VM5 | Observability | QA | DONE | | **DONE** – diagnostics meter exposes fetch/parse/map metrics and structured logs. | +| VM6 | SourceState + hash dedupe | Conn | DONE | Storage | **DONE** – fetch cache stores sha/etag to skip unchanged advisories during resume. | +| VM6a | Options & HttpClient configuration | Conn | DONE | Source.Common | **DONE** – `AddVmwareConnector` configures allowlisted HttpClient + options. | +| VM7 | Dependency injection routine & scheduler registration | Conn | DONE | Core | **DONE** – `VmwareDependencyInjectionRoutine` registers fetch/parse/map jobs. | +| VM8 | Replace stub plugin with connector pipeline skeleton | Conn | DONE | Source.Common | **DONE** – connector implements fetch/parse/map persisting docs, DTOs, advisories. | +| VM9 | Range primitives + provenance diagnostics refresh | Conn | DONE | Models, Storage.Mongo | Vendor primitives emitted (SemVer + vendor extensions), provenance tags/logging updated, snapshots refreshed. | ## Changelog - YYYY-MM-DD: Created. diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs index 3ba2db7b..1a18fa03 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs @@ -18,6 +18,7 @@ using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.Storage.Mongo.Advisories; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; using StellaOps.Plugin; namespace StellaOps.Feedser.Source.Vndr.Vmware; @@ -37,8 +38,10 @@ public sealed class VmwareConnector : IFeedConnector private readonly IDtoStore _dtoStore; private readonly IAdvisoryStore _advisoryStore; private readonly ISourceStateRepository _stateRepository; + private readonly IPsirtFlagStore _psirtFlagStore; private readonly VmwareOptions _options; private readonly TimeProvider _timeProvider; + private readonly VmwareDiagnostics _diagnostics; private readonly ILogger _logger; public VmwareConnector( @@ -49,8 +52,10 @@ public sealed class VmwareConnector : IFeedConnector IDtoStore dtoStore, IAdvisoryStore advisoryStore, ISourceStateRepository stateRepository, + IPsirtFlagStore psirtFlagStore, IOptions options, TimeProvider? timeProvider, + VmwareDiagnostics diagnostics, ILogger logger) { _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); @@ -60,9 +65,11 @@ public sealed class VmwareConnector : IFeedConnector _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); _options.Validate(); _timeProvider = timeProvider ?? TimeProvider.System; + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -75,6 +82,9 @@ public sealed class VmwareConnector : IFeedConnector var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); var now = _timeProvider.GetUtcNow(); var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var fetchCache = new Dictionary(cursor.FetchCache, StringComparer.OrdinalIgnoreCase); + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); var remainingCapacity = _options.MaxAdvisoriesPerFetch; IReadOnlyList indexItems; @@ -84,6 +94,7 @@ public sealed class VmwareConnector : IFeedConnector } catch (Exception ex) { + _diagnostics.FetchFailure(); _logger.LogError(ex, "Failed to retrieve VMware advisory index"); await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); throw; @@ -101,6 +112,8 @@ public sealed class VmwareConnector : IFeedConnector .ToArray(); var baseline = cursor.LastModified ?? now - _options.InitialBackfill; + var resumeStart = baseline - _options.ModifiedTolerance; + ProvenanceDiagnostics.ReportResumeWindow(SourceName, resumeStart, _logger); var processedIds = new HashSet(cursor.ProcessedIds, StringComparer.OrdinalIgnoreCase); var maxModified = cursor.LastModified ?? DateTimeOffset.MinValue; var processedUpdated = false; @@ -130,6 +143,16 @@ public sealed class VmwareConnector : IFeedConnector continue; } + if (!Uri.TryCreate(item.DetailUrl, UriKind.Absolute, out var detailUri)) + { + _logger.LogWarning("VMware advisory {AdvisoryId} has invalid detail URL {Url}", item.Id, item.DetailUrl); + continue; + } + + var cacheKey = detailUri.AbsoluteUri; + touchedResources.Add(cacheKey); + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, cacheKey, cancellationToken).ConfigureAwait(false); var metadata = new Dictionary(StringComparer.Ordinal) { ["vmware.id"] = item.Id, @@ -140,14 +163,18 @@ public sealed class VmwareConnector : IFeedConnector try { result = await _fetchService.FetchAsync( - new SourceFetchRequest(VmwareOptions.HttpClientName, SourceName, new Uri(item.DetailUrl)) + new SourceFetchRequest(VmwareOptions.HttpClientName, SourceName, detailUri) { Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "application/json" }, }, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { + _diagnostics.FetchFailure(); _logger.LogError(ex, "Failed to fetch VMware advisory {AdvisoryId}", item.Id); await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); throw; @@ -155,15 +182,24 @@ public sealed class VmwareConnector : IFeedConnector if (result.IsNotModified) { + _diagnostics.FetchUnchanged(); + if (existing is not null) + { + fetchCache[cacheKey] = VmwareFetchCacheEntry.FromDocument(existing); + pendingDocuments.Remove(existing.Id); + pendingMappings.Remove(existing.Id); + _logger.LogInformation("VMware advisory {AdvisoryId} returned 304 Not Modified", item.Id); + } + continue; } if (!result.IsSuccess || result.Document is null) { + _diagnostics.FetchFailure(); continue; } - pendingDocuments.Add(result.Document.Id); remainingCapacity--; if (modified > maxModified) @@ -179,6 +215,31 @@ public sealed class VmwareConnector : IFeedConnector processedUpdated = true; } + var cacheEntry = VmwareFetchCacheEntry.FromDocument(result.Document); + + if (existing is not null + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal) + && cursor.TryGetFetchCache(cacheKey, out var cachedEntry) + && cachedEntry.Matches(result.Document)) + { + _diagnostics.FetchUnchanged(); + fetchCache[cacheKey] = cacheEntry; + pendingDocuments.Remove(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("VMware advisory {AdvisoryId} unchanged; skipping reprocessing", item.Id); + continue; + } + + _diagnostics.FetchItem(); + fetchCache[cacheKey] = cacheEntry; + pendingDocuments.Add(result.Document.Id); + _logger.LogInformation( + "VMware advisory {AdvisoryId} fetched (documentId={DocumentId}, sha256={Sha})", + item.Id, + result.Document.Id, + result.Document.Sha256); + if (_options.RequestDelay > TimeSpan.Zero) { try @@ -192,9 +253,19 @@ public sealed class VmwareConnector : IFeedConnector } } + if (fetchCache.Count > 0 && touchedResources.Count > 0) + { + var stale = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in stale) + { + fetchCache.Remove(key); + } + } + var updatedCursor = cursor .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(cursor.PendingMappings); + .WithPendingMappings(pendingMappings) + .WithFetchCache(fetchCache); if (processedUpdated) { @@ -233,6 +304,7 @@ public sealed class VmwareConnector : IFeedConnector _logger.LogWarning("VMware document {DocumentId} missing GridFS payload", document.Id); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); remaining.Remove(documentId); + _diagnostics.ParseFailure(); continue; } @@ -257,6 +329,7 @@ public sealed class VmwareConnector : IFeedConnector _logger.LogWarning(ex, "Failed to deserialize VMware advisory {DocumentId}", document.Id); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); remaining.Remove(documentId); + _diagnostics.ParseFailure(); continue; } @@ -265,6 +338,7 @@ public sealed class VmwareConnector : IFeedConnector _logger.LogWarning("VMware advisory document {DocumentId} contained empty payload", document.Id); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); remaining.Remove(documentId); + _diagnostics.ParseFailure(); continue; } @@ -338,9 +412,15 @@ public sealed class VmwareConnector : IFeedConnector continue; } - var advisory = VmwareMapper.Map(detail, document, dto); + var (advisory, flag) = VmwareMapper.Map(detail, document, dto); await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + _diagnostics.MapAffectedCount(advisory.AffectedPackages.Length); + _logger.LogInformation( + "VMware advisory {AdvisoryId} mapped with {AffectedCount} affected packages", + detail.AdvisoryId, + advisory.AffectedPackages.Length); pendingMappings.Remove(documentId); } diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs new file mode 100644 index 00000000..9d7cd687 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs @@ -0,0 +1,67 @@ +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +/// +/// VMware connector metrics (fetch, parse, map). +/// +public sealed class VmwareDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Vndr.Vmware"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchItems; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseFailures; + private readonly Histogram _mapAffectedCount; + + public VmwareDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchItems = _meter.CreateCounter( + name: "vmware.fetch.items", + unit: "documents", + description: "Number of VMware advisory documents fetched."); + _fetchFailures = _meter.CreateCounter( + name: "vmware.fetch.failures", + unit: "operations", + description: "Number of VMware fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "vmware.fetch.unchanged", + unit: "documents", + description: "Number of VMware advisories skipped due to unchanged content."); + _parseFailures = _meter.CreateCounter( + name: "vmware.parse.fail", + unit: "documents", + description: "Number of VMware advisory documents that failed to parse."); + _mapAffectedCount = _meter.CreateHistogram( + name: "vmware.map.affected_count", + unit: "packages", + description: "Distribution of affected-package counts emitted per VMware advisory."); + } + + public void FetchItem() => _fetchItems.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void MapAffectedCount(int count) + { + if (count < 0) + { + return; + } + + _mapAffectedCount.Record(count); + } + + public Meter Meter => _meter; + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs index 32018526..dea0f22d 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using System; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; using StellaOps.Feedser.Source.Common.Http; using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; @@ -28,6 +29,7 @@ public static class VmwareServiceCollectionExtensions clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; }); + services.TryAddSingleton(); services.AddTransient(); services.AddTransient(); services.AddTransient(); diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs index 0639e096..75597a55 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Options; using StellaOps.Feedser.Models; using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; using StellaOps.Feedser.Storage.Mongo.Migrations; using Xunit; using Xunit.Abstractions; @@ -60,7 +61,8 @@ public sealed class AdvisoryStorePerformanceTests : IClassFixture.Instance); + var aliasStore = new AliasStore(database, NullLogger.Instance); + var store = new AdvisoryStore(database, aliasStore, NullLogger.Instance, TimeProvider.System); using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(45)); var advisories = Enumerable.Range(0, LargeAdvisoryCount) diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs index 848f536f..856043b3 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs @@ -1,6 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Linq; using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; using StellaOps.Feedser.Models; using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; namespace StellaOps.Feedser.Storage.Mongo.Tests; @@ -17,7 +22,11 @@ public sealed class AdvisoryStoreTests : IClassFixture [Fact] public async Task UpsertAndFetchAdvisory() { - var store = new AdvisoryStore(_fixture.Database, NullLogger.Instance); + await DropCollectionAsync(MongoStorageDefaults.Collections.Advisory); + await DropCollectionAsync(MongoStorageDefaults.Collections.Alias); + + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var store = new AdvisoryStore(_fixture.Database, aliasStore, NullLogger.Instance, TimeProvider.System); var advisory = new Advisory( advisoryKey: "ADV-1", title: "Sample Advisory", @@ -41,5 +50,108 @@ public sealed class AdvisoryStoreTests : IClassFixture var recent = await store.GetRecentAsync(5, CancellationToken.None); Assert.NotEmpty(recent); + + var aliases = await aliasStore.GetByAdvisoryAsync("ADV-1", CancellationToken.None); + Assert.Contains(aliases, record => record.Scheme == AliasStoreConstants.PrimaryScheme && record.Value == "ADV-1"); + Assert.Contains(aliases, record => record.Value == "ALIAS-1"); + } + + [Fact] + public async Task RangePrimitives_RoundTripThroughMongo() + { + await DropCollectionAsync(MongoStorageDefaults.Collections.Advisory); + await DropCollectionAsync(MongoStorageDefaults.Collections.Alias); + + var aliasStore = new AliasStore(_fixture.Database, NullLogger.Instance); + var store = new AdvisoryStore(_fixture.Database, aliasStore, NullLogger.Instance, TimeProvider.System); + + var recordedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero); + var provenance = new AdvisoryProvenance("source-x", "mapper", "payload-123", recordedAt); + var rangePrimitives = new RangePrimitives( + new SemVerPrimitive( + Introduced: "1.0.0", + IntroducedInclusive: true, + Fixed: "1.2.0", + FixedInclusive: false, + LastAffected: "1.1.5", + LastAffectedInclusive: true, + ConstraintExpression: ">=1.0.0 <1.2.0"), + new NevraPrimitive( + Introduced: new NevraComponent("pkg", 0, "1.0.0", "1", "x86_64"), + Fixed: new NevraComponent("pkg", 1, "1.2.0", "2", "x86_64"), + LastAffected: null), + new EvrPrimitive( + Introduced: new EvrComponent(1, "1.0.0", "1"), + Fixed: null, + LastAffected: new EvrComponent(1, "1.1.5", null)), + new Dictionary(StringComparer.Ordinal) + { + ["channel"] = "stable", + ["notesHash"] = "abc123", + }); + + var versionRange = new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "1.0.0", + fixedVersion: "1.2.0", + lastAffectedVersion: "1.1.5", + rangeExpression: ">=1.0.0 <1.2.0", + provenance, + rangePrimitives); + + var affectedPackage = new AffectedPackage( + type: "semver", + identifier: "pkg@1.x", + platform: "linux", + versionRanges: new[] { versionRange }, + statuses: Array.Empty(), + provenance: new[] { provenance }); + + var advisory = new Advisory( + advisoryKey: "ADV-RANGE-1", + title: "Sample Range Primitive", + summary: "Testing range primitive persistence.", + language: "en", + published: recordedAt, + modified: recordedAt, + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-0001" }, + references: Array.Empty(), + affectedPackages: new[] { affectedPackage }, + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + + await store.UpsertAsync(advisory, CancellationToken.None); + + var fetched = await store.FindAsync("ADV-RANGE-1", CancellationToken.None); + Assert.NotNull(fetched); + var fetchedPackage = Assert.Single(fetched!.AffectedPackages); + var fetchedRange = Assert.Single(fetchedPackage.VersionRanges); + + Assert.Equal(versionRange.RangeKind, fetchedRange.RangeKind); + Assert.Equal(versionRange.IntroducedVersion, fetchedRange.IntroducedVersion); + Assert.Equal(versionRange.FixedVersion, fetchedRange.FixedVersion); + Assert.Equal(versionRange.LastAffectedVersion, fetchedRange.LastAffectedVersion); + Assert.Equal(versionRange.RangeExpression, fetchedRange.RangeExpression); + Assert.Equal(versionRange.Provenance, fetchedRange.Provenance); + + Assert.NotNull(fetchedRange.Primitives); + Assert.Equal(rangePrimitives.SemVer, fetchedRange.Primitives!.SemVer); + Assert.Equal(rangePrimitives.Nevra, fetchedRange.Primitives.Nevra); + Assert.Equal(rangePrimitives.Evr, fetchedRange.Primitives.Evr); + Assert.Equal(rangePrimitives.VendorExtensions, fetchedRange.Primitives.VendorExtensions); + } + + private async Task DropCollectionAsync(string collectionName) + { + try + { + await _fixture.Database.DropCollectionAsync(collectionName); + } + catch (MongoDB.Driver.MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + // ignore missing collection + } } } diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs new file mode 100644 index 00000000..3229bdfb --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Aliases; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class AliasStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public AliasStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task ReplaceAsync_UpsertsAliases_AndDetectsCollision() + { + await DropAliasCollectionAsync(); + var store = new AliasStore(_fixture.Database, NullLogger.Instance); + + var timestamp = DateTimeOffset.UtcNow; + await store.ReplaceAsync( + "ADV-1", + new[] { new AliasEntry("CVE", "CVE-2025-1234"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-1") }, + timestamp, + CancellationToken.None); + + var firstAliases = await store.GetByAdvisoryAsync("ADV-1", CancellationToken.None); + Assert.Contains(firstAliases, record => record.Scheme == "CVE" && record.Value == "CVE-2025-1234"); + + var result = await store.ReplaceAsync( + "ADV-2", + new[] { new AliasEntry("CVE", "CVE-2025-1234"), new AliasEntry(AliasStoreConstants.PrimaryScheme, "ADV-2") }, + timestamp.AddMinutes(1), + CancellationToken.None); + + Assert.NotEmpty(result.Collisions); + var collision = Assert.Single(result.Collisions); + Assert.Equal("CVE", collision.Scheme); + Assert.Contains("ADV-1", collision.AdvisoryKeys); + Assert.Contains("ADV-2", collision.AdvisoryKeys); + } + + private async Task DropAliasCollectionAsync() + { + try + { + await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.Alias); + } + catch (MongoDB.Driver.MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs index 6e023e46..ea5d5342 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs @@ -23,6 +23,7 @@ public sealed class ExportStateManagerTests targetRepository: "registry.local/json", exporterVersion: "1.0.0", resetBaseline: true, + manifest: Array.Empty(), cancellationToken: CancellationToken.None); Assert.Equal("export:json", record.Id); @@ -51,6 +52,7 @@ public sealed class ExportStateManagerTests targetRepository: null, exporterVersion: "1.0.0", resetBaseline: true, + manifest: Array.Empty(), cancellationToken: CancellationToken.None); timeProvider.Advance(TimeSpan.FromMinutes(5)); @@ -62,6 +64,7 @@ public sealed class ExportStateManagerTests targetRepository: null, exporterVersion: "1.0.1", resetBaseline: false, + manifest: Array.Empty(), cancellationToken: CancellationToken.None); Assert.Equal("20240720T120000Z", withoutReset.BaseExportId); @@ -79,6 +82,7 @@ public sealed class ExportStateManagerTests targetRepository: null, exporterVersion: "1.0.2", resetBaseline: true, + manifest: Array.Empty(), cancellationToken: CancellationToken.None); Assert.Equal("20240720T121000Z", reset.BaseExportId); @@ -104,6 +108,7 @@ public sealed class ExportStateManagerTests targetRepository: "registry/v1/json", exporterVersion: "1.0.0", resetBaseline: true, + manifest: Array.Empty(), cancellationToken: CancellationToken.None); timeProvider.Advance(TimeSpan.FromMinutes(10)); @@ -115,6 +120,7 @@ public sealed class ExportStateManagerTests targetRepository: "registry/v2/json", exporterVersion: "1.1.0", resetBaseline: false, + manifest: Array.Empty(), cancellationToken: CancellationToken.None); Assert.Equal("20240721T081000Z", updated.BaseExportId); @@ -134,6 +140,7 @@ public sealed class ExportStateManagerTests deltaDigest: "sha256:def", cursor: null, exporterVersion: "1.0.1", + manifest: Array.Empty(), cancellationToken: CancellationToken.None)); } @@ -152,6 +159,7 @@ public sealed class ExportStateManagerTests targetRepository: null, exporterVersion: "1.0.0", resetBaseline: true, + manifest: Array.Empty(), cancellationToken: CancellationToken.None); timeProvider.Advance(TimeSpan.FromMinutes(10)); @@ -160,6 +168,7 @@ public sealed class ExportStateManagerTests deltaDigest: "sha256:ef01", cursor: "cursor-2", exporterVersion: "1.0.1", + manifest: Array.Empty(), cancellationToken: CancellationToken.None); Assert.Equal("sha256:ef01", delta.LastDeltaDigest); diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs index 4f88562d..dc3a9207 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs @@ -1,3 +1,4 @@ +using System; using Microsoft.Extensions.Logging.Abstractions; using StellaOps.Feedser.Storage.Mongo.Exporting; @@ -26,13 +27,16 @@ public sealed class ExportStateStoreTests : IClassFixture()); var saved = await store.UpsertAsync(record, CancellationToken.None); Assert.Equal("json", saved.Id); + Assert.Empty(saved.Files); var fetched = await store.FindAsync("json", CancellationToken.None); Assert.NotNull(fetched); Assert.Equal("sha-full", fetched!.LastFullDigest); + Assert.Empty(fetched.Files); } } diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs index 271951ed..fc0875ef 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs @@ -18,6 +18,7 @@ public sealed class MongoJobStoreTests : IClassFixture [Fact] public async Task CreateStartCompleteLifecycle() { + await ResetCollectionAsync(); var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); var store = new MongoJobStore(collection, NullLogger.Instance); @@ -41,9 +42,9 @@ public sealed class MongoJobStoreTests : IClassFixture Assert.NotNull(completed); Assert.Equal(JobRunStatus.Succeeded, completed!.Status); - var recent = await store.GetRecentRunsAsync(null, 10, CancellationToken.None); - Assert.Single(recent); - Assert.Equal(JobRunStatus.Succeeded, recent[0].Status); + var recent = await store.GetRecentRunsAsync("mongo:test", 10, CancellationToken.None); + var snapshot = Assert.Single(recent); + Assert.Equal(JobRunStatus.Succeeded, snapshot.Status); var active = await store.GetActiveRunsAsync(CancellationToken.None); Assert.Empty(active); @@ -56,6 +57,7 @@ public sealed class MongoJobStoreTests : IClassFixture [Fact] public async Task StartAndFailRunHonorsStateTransitions() { + await ResetCollectionAsync(); var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); var store = new MongoJobStore(collection, NullLogger.Instance); @@ -89,6 +91,7 @@ public sealed class MongoJobStoreTests : IClassFixture [Fact] public async Task CompletingUnknownRunReturnsNull() { + await ResetCollectionAsync(); var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); var store = new MongoJobStore(collection, NullLogger.Instance); @@ -96,5 +99,15 @@ public sealed class MongoJobStoreTests : IClassFixture Assert.Null(result); } -} + private async Task ResetCollectionAsync() + { + try + { + await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.Jobs); + } + catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs index 1c78adc5..06722591 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs @@ -40,7 +40,7 @@ public sealed class RawDocumentRetentionServiceTests : IClassFixture d.DocumentId == expiredId)); Assert.Equal(1, await documents.CountDocumentsAsync(d => d.Id == freshId)); - using var cursor = await bucket.FindAsync(Builders.Filter.Eq(info => info.Id, gridFsId)); + var filter = Builders.Filter.Eq("_id", gridFsId); + using var cursor = await bucket.FindAsync(filter); Assert.Empty(await cursor.ToListAsync()); } } diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs index 3a5ead5a..4de24ad4 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs @@ -1,3 +1,6 @@ +using System; +using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; using System.Text.Json; using System.Text.Json.Serialization; @@ -5,6 +8,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Bson; using MongoDB.Driver; using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Aliases; namespace StellaOps.Feedser.Storage.Mongo.Advisories; @@ -12,12 +16,20 @@ public sealed class AdvisoryStore : IAdvisoryStore { private readonly IMongoCollection _collection; private readonly ILogger _logger; + private readonly IAliasStore _aliasStore; + private readonly TimeProvider _timeProvider; - public AdvisoryStore(IMongoDatabase database, ILogger logger) + public AdvisoryStore( + IMongoDatabase database, + IAliasStore aliasStore, + ILogger logger, + TimeProvider? timeProvider = null) { _collection = (database ?? throw new ArgumentNullException(nameof(database))) .GetCollection(MongoStorageDefaults.Collections.Advisory); + _aliasStore = aliasStore ?? throw new ArgumentNullException(nameof(aliasStore)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; } @@ -25,6 +37,19 @@ public sealed class AdvisoryStore : IAdvisoryStore { ArgumentNullException.ThrowIfNull(advisory); + var missing = ProvenanceInspector.FindMissingProvenance(advisory); + var primarySource = advisory.Provenance.FirstOrDefault()?.Source ?? "unknown"; + foreach (var item in missing) + { + var source = string.IsNullOrWhiteSpace(item.Source) ? primarySource : item.Source; + _logger.LogWarning( + "Missing provenance detected for {Component} in advisory {AdvisoryKey} (source {Source}).", + item.Component, + advisory.AdvisoryKey, + source); + ProvenanceDiagnostics.RecordMissing(source, item.Component, item.RecordedAt); + } + var payload = CanonicalJsonSerializer.Serialize(advisory); var document = new AdvisoryDocument { @@ -37,6 +62,10 @@ public sealed class AdvisoryStore : IAdvisoryStore var options = new ReplaceOptions { IsUpsert = true }; await _collection.ReplaceOneAsync(x => x.AdvisoryKey == advisory.AdvisoryKey, document, options, cancellationToken).ConfigureAwait(false); _logger.LogDebug("Upserted advisory {AdvisoryKey}", advisory.AdvisoryKey); + + var aliasEntries = BuildAliasEntries(advisory); + var updatedAt = _timeProvider.GetUtcNow(); + await _aliasStore.ReplaceAsync(advisory.AdvisoryKey, aliasEntries, updatedAt, cancellationToken).ConfigureAwait(false); } public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) @@ -49,6 +78,23 @@ public sealed class AdvisoryStore : IAdvisoryStore return document is null ? null : Deserialize(document.Payload); } + private static IEnumerable BuildAliasEntries(Advisory advisory) + { + foreach (var alias in advisory.Aliases) + { + if (AliasSchemeRegistry.TryGetScheme(alias, out var scheme)) + { + yield return new AliasEntry(scheme, alias); + } + else + { + yield return new AliasEntry(AliasStoreConstants.UnscopedScheme, alias); + } + } + + yield return new AliasEntry(AliasStoreConstants.PrimaryScheme, advisory.AdvisoryKey); + } + public async Task> GetRecentAsync(int limit, CancellationToken cancellationToken) { var cursor = await _collection.Find(FilterDefinition.Empty) @@ -182,8 +228,13 @@ public sealed class AdvisoryStore : IAdvisoryStore var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument ? DeserializeProvenance(provenanceValue.AsBsonDocument) : AdvisoryProvenance.Empty; + RangePrimitives? primitives = null; + if (document.TryGetValue("primitives", out var primitivesValue) && primitivesValue.IsBsonDocument) + { + primitives = DeserializePrimitives(primitivesValue.AsBsonDocument); + } - return new AffectedVersionRange(rangeKind, introducedVersion, fixedVersion, lastAffectedVersion, rangeExpression, provenance); + return new AffectedVersionRange(rangeKind, introducedVersion, fixedVersion, lastAffectedVersion, rangeExpression, provenance, primitives); } private static AffectedPackageStatus DeserializeStatus(BsonDocument document) @@ -225,6 +276,104 @@ public sealed class AdvisoryStore : IAdvisoryStore return new AdvisoryProvenance(source, kind, value ?? string.Empty, recordedAt ?? DateTimeOffset.UtcNow); } + private static RangePrimitives? DeserializePrimitives(BsonDocument document) + { + SemVerPrimitive? semVer = null; + NevraPrimitive? nevra = null; + EvrPrimitive? evr = null; + IReadOnlyDictionary? vendor = null; + + if (document.TryGetValue("semVer", out var semverValue) && semverValue.IsBsonDocument) + { + var semverDoc = semverValue.AsBsonDocument; + semVer = new SemVerPrimitive( + semverDoc.TryGetValue("introduced", out var semIntroduced) && semIntroduced.IsString ? semIntroduced.AsString : null, + semverDoc.TryGetValue("introducedInclusive", out var semIntroducedInclusive) && semIntroducedInclusive.IsBoolean && semIntroducedInclusive.AsBoolean, + semverDoc.TryGetValue("fixed", out var semFixed) && semFixed.IsString ? semFixed.AsString : null, + semverDoc.TryGetValue("fixedInclusive", out var semFixedInclusive) && semFixedInclusive.IsBoolean && semFixedInclusive.AsBoolean, + semverDoc.TryGetValue("lastAffected", out var semLast) && semLast.IsString ? semLast.AsString : null, + semverDoc.TryGetValue("lastAffectedInclusive", out var semLastInclusive) && semLastInclusive.IsBoolean && semLastInclusive.AsBoolean, + semverDoc.TryGetValue("constraintExpression", out var constraint) && constraint.IsString ? constraint.AsString : null); + } + + if (document.TryGetValue("nevra", out var nevraValue) && nevraValue.IsBsonDocument) + { + var nevraDoc = nevraValue.AsBsonDocument; + nevra = new NevraPrimitive( + DeserializeNevraComponent(nevraDoc, "introduced"), + DeserializeNevraComponent(nevraDoc, "fixed"), + DeserializeNevraComponent(nevraDoc, "lastAffected")); + } + + if (document.TryGetValue("evr", out var evrValue) && evrValue.IsBsonDocument) + { + var evrDoc = evrValue.AsBsonDocument; + evr = new EvrPrimitive( + DeserializeEvrComponent(evrDoc, "introduced"), + DeserializeEvrComponent(evrDoc, "fixed"), + DeserializeEvrComponent(evrDoc, "lastAffected")); + } + + if (document.TryGetValue("vendorExtensions", out var vendorValue) && vendorValue.IsBsonDocument) + { + vendor = vendorValue.AsBsonDocument.Elements + .Where(static e => e.Value.IsString) + .ToDictionary(static e => e.Name, static e => e.Value.AsString, StringComparer.Ordinal); + if (vendor.Count == 0) + { + vendor = null; + } + } + + if (semVer is null && nevra is null && evr is null && vendor is null) + { + return null; + } + + return new RangePrimitives(semVer, nevra, evr, vendor); + } + + private static NevraComponent? DeserializeNevraComponent(BsonDocument parent, string field) + { + if (!parent.TryGetValue(field, out var value) || !value.IsBsonDocument) + { + return null; + } + + var component = value.AsBsonDocument; + var name = component.TryGetValue("name", out var nameValue) && nameValue.IsString ? nameValue.AsString : null; + var version = component.TryGetValue("version", out var versionValue) && versionValue.IsString ? versionValue.AsString : null; + if (name is null || version is null) + { + return null; + } + + var epoch = component.TryGetValue("epoch", out var epochValue) && epochValue.IsNumeric ? epochValue.ToInt32() : 0; + var release = component.TryGetValue("release", out var releaseValue) && releaseValue.IsString ? releaseValue.AsString : string.Empty; + var architecture = component.TryGetValue("architecture", out var archValue) && archValue.IsString ? archValue.AsString : null; + + return new NevraComponent(name, epoch, version, release, architecture); + } + + private static EvrComponent? DeserializeEvrComponent(BsonDocument parent, string field) + { + if (!parent.TryGetValue(field, out var value) || !value.IsBsonDocument) + { + return null; + } + + var component = value.AsBsonDocument; + var epoch = component.TryGetValue("epoch", out var epochValue) && epochValue.IsNumeric ? epochValue.ToInt32() : 0; + var upstream = component.TryGetValue("upstreamVersion", out var upstreamValue) && upstreamValue.IsString ? upstreamValue.AsString : null; + if (upstream is null) + { + return null; + } + + var revision = component.TryGetValue("revision", out var revisionValue) && revisionValue.IsString ? revisionValue.AsString : null; + return new EvrComponent(epoch, upstream, revision); + } + private static DateTimeOffset? TryReadDateTime(BsonDocument document, string field) => document.TryGetValue(field, out var value) ? TryConvertDateTime(value) : null; diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs new file mode 100644 index 00000000..53d73f96 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs @@ -0,0 +1,38 @@ +using System; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +[BsonIgnoreExtraElements] +internal sealed class AliasDocument +{ + [BsonId] + public ObjectId Id { get; set; } + + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("scheme")] + public string Scheme { get; set; } = string.Empty; + + [BsonElement("value")] + public string Value { get; set; } = string.Empty; + + [BsonElement("updatedAt")] + public DateTime UpdatedAt { get; set; } +} + +internal static class AliasDocumentExtensions +{ + public static AliasRecord ToRecord(this AliasDocument document) + { + ArgumentNullException.ThrowIfNull(document); + var updatedAt = DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc); + return new AliasRecord( + document.AdvisoryKey, + document.Scheme, + document.Value, + new DateTimeOffset(updatedAt)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs new file mode 100644 index 00000000..a63139d2 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs @@ -0,0 +1,157 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +public sealed class AliasStore : IAliasStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public AliasStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.Alias); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ReplaceAsync( + string advisoryKey, + IEnumerable aliases, + DateTimeOffset updatedAt, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); + + var aliasList = Normalize(aliases).ToArray(); + var deleteFilter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + await _collection.DeleteManyAsync(deleteFilter, cancellationToken).ConfigureAwait(false); + + if (aliasList.Length > 0) + { + var documents = new List(aliasList.Length); + var updatedAtUtc = updatedAt.ToUniversalTime().UtcDateTime; + foreach (var alias in aliasList) + { + documents.Add(new AliasDocument + { + Id = ObjectId.GenerateNewId(), + AdvisoryKey = advisoryKey, + Scheme = alias.Scheme, + Value = alias.Value, + UpdatedAt = updatedAtUtc, + }); + } + + if (documents.Count > 0) + { + await _collection.InsertManyAsync( + documents, + new InsertManyOptions { IsOrdered = false }, + cancellationToken).ConfigureAwait(false); + } + } + + if (aliasList.Length == 0) + { + return new AliasUpsertResult(advisoryKey, Array.Empty()); + } + + var collisions = new List(); + foreach (var alias in aliasList) + { + var filter = Builders.Filter.Eq(x => x.Scheme, alias.Scheme) + & Builders.Filter.Eq(x => x.Value, alias.Value); + + using var cursor = await _collection.FindAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false); + var advisoryKeys = new HashSet(StringComparer.OrdinalIgnoreCase); + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var document in cursor.Current) + { + advisoryKeys.Add(document.AdvisoryKey); + } + } + + if (advisoryKeys.Count <= 1) + { + continue; + } + + var collision = new AliasCollision(alias.Scheme, alias.Value, advisoryKeys.ToArray()); + collisions.Add(collision); + AliasStoreMetrics.RecordCollision(alias.Scheme, advisoryKeys.Count); + _logger.LogWarning( + "Alias collision detected for {Scheme}:{Value}; advisories: {Advisories}", + alias.Scheme, + alias.Value, + string.Join(", ", advisoryKeys)); + } + + return new AliasUpsertResult(advisoryKey, collisions); + } + + public async Task> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(scheme); + ArgumentException.ThrowIfNullOrWhiteSpace(value); + + var normalizedScheme = NormalizeScheme(scheme); + var normalizedValue = value.Trim(); + var filter = Builders.Filter.Eq(x => x.Scheme, normalizedScheme) + & Builders.Filter.Eq(x => x.Value, normalizedValue); + + var documents = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); + return documents.Select(static d => d.ToRecord()).ToArray(); + } + + public async Task> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + var documents = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); + return documents.Select(static d => d.ToRecord()).ToArray(); + } + + private static IEnumerable Normalize(IEnumerable aliases) + { + if (aliases is null) + { + yield break; + } + + var seen = new HashSet(StringComparer.Ordinal); + foreach (var alias in aliases) + { + if (alias is null) + { + continue; + } + + var scheme = NormalizeScheme(alias.Scheme); + var value = alias.Value?.Trim(); + if (string.IsNullOrEmpty(value)) + { + continue; + } + + var key = $"{scheme}\u0001{value}"; + if (!seen.Add(key)) + { + continue; + } + + yield return new AliasEntry(scheme, value); + } + } + + private static string NormalizeScheme(string scheme) + { + return string.IsNullOrWhiteSpace(scheme) + ? AliasStoreConstants.UnscopedScheme + : scheme.Trim().ToUpperInvariant(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs new file mode 100644 index 00000000..babf0719 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +public static class AliasStoreConstants +{ + public const string PrimaryScheme = "PRIMARY"; + public const string UnscopedScheme = "UNSCOPED"; +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs new file mode 100644 index 00000000..a5ad2271 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +internal static class AliasStoreMetrics +{ + private static readonly Meter Meter = new("StellaOps.Feedser.Merge"); + + internal static readonly Counter AliasCollisionCounter = Meter.CreateCounter( + "feedser.merge.alias_conflict", + unit: "count", + description: "Number of alias collisions detected when the same alias maps to multiple advisories."); + + public static void RecordCollision(string scheme, int advisoryCount) + { + AliasCollisionCounter.Add( + 1, + new KeyValuePair("scheme", scheme), + new KeyValuePair("advisory_count", advisoryCount)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs new file mode 100644 index 00000000..aa48a8c2 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.Aliases; + +public interface IAliasStore +{ + Task ReplaceAsync( + string advisoryKey, + IEnumerable aliases, + DateTimeOffset updatedAt, + CancellationToken cancellationToken); + + Task> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken); + + Task> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken); +} + +public sealed record AliasEntry(string Scheme, string Value); + +public sealed record AliasRecord(string AdvisoryKey, string Scheme, string Value, DateTimeOffset UpdatedAt); + +public sealed record AliasCollision(string Scheme, string Value, IReadOnlyList AdvisoryKeys); + +public sealed record AliasUpsertResult(string AdvisoryKey, IReadOnlyList Collisions); diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs index 446820b2..454bab72 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs @@ -9,7 +9,7 @@ namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; public sealed class ChangeHistoryDocument { [BsonId] - public Guid Id { get; set; } + public string Id { get; set; } = string.Empty; [BsonElement("source")] public string SourceName { get; set; } = string.Empty; @@ -18,7 +18,7 @@ public sealed class ChangeHistoryDocument public string AdvisoryKey { get; set; } = string.Empty; [BsonElement("documentId")] - public Guid DocumentId { get; set; } + public string DocumentId { get; set; } = string.Empty; [BsonElement("documentSha256")] public string DocumentSha256 { get; set; } = string.Empty; diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs index 3acaf472..edb94071 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs @@ -22,10 +22,10 @@ internal static class ChangeHistoryDocumentExtensions return new ChangeHistoryDocument { - Id = record.Id, + Id = record.Id.ToString(), SourceName = record.SourceName, AdvisoryKey = record.AdvisoryKey, - DocumentId = record.DocumentId, + DocumentId = record.DocumentId.ToString(), DocumentSha256 = record.DocumentSha256, CurrentHash = record.CurrentHash, PreviousHash = record.PreviousHash, @@ -55,10 +55,10 @@ internal static class ChangeHistoryDocumentExtensions var capturedAtUtc = DateTime.SpecifyKind(document.CapturedAt, DateTimeKind.Utc); return new ChangeHistoryRecord( - document.Id, + Guid.Parse(document.Id), document.SourceName, document.AdvisoryKey, - document.DocumentId, + Guid.Parse(document.DocumentId), document.DocumentSha256, document.CurrentHash, document.PreviousHash, diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs index 26bf3f4f..043ac51e 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs @@ -1,3 +1,4 @@ +using System; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; @@ -7,7 +8,7 @@ namespace StellaOps.Feedser.Storage.Mongo.Documents; public sealed class DocumentDocument { [BsonId] - public Guid Id { get; set; } + public string Id { get; set; } = string.Empty; [BsonElement("sourceName")] public string SourceName { get; set; } = string.Empty; @@ -59,7 +60,7 @@ internal static class DocumentDocumentExtensions { return new DocumentDocument { - Id = record.Id, + Id = record.Id.ToString(), SourceName = record.SourceName, Uri = record.Uri, FetchedAt = record.FetchedAt.UtcDateTime, @@ -96,7 +97,7 @@ internal static class DocumentDocumentExtensions } return new DocumentRecord( - document.Id, + Guid.Parse(document.Id), document.SourceName, document.Uri, DateTime.SpecifyKind(document.FetchedAt, DateTimeKind.Utc), diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs index 8ae37335..51c4665a 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs @@ -48,7 +48,8 @@ public sealed class DocumentStore : IDocumentStore public async Task FindAsync(Guid id, CancellationToken cancellationToken) { - var document = await _collection.Find(x => x.Id == id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + var idValue = id.ToString(); + var document = await _collection.Find(x => x.Id == idValue).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); return document?.ToRecord(); } @@ -60,7 +61,8 @@ public sealed class DocumentStore : IDocumentStore .Set(x => x.Status, status) .Set(x => x.LastModified, DateTime.UtcNow); - var result = await _collection.UpdateOneAsync(x => x.Id == id, update, cancellationToken: cancellationToken).ConfigureAwait(false); + var idValue = id.ToString(); + var result = await _collection.UpdateOneAsync(x => x.Id == idValue, update, cancellationToken: cancellationToken).ConfigureAwait(false); return result.MatchedCount > 0; } } diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs index f1317c08..15b0ee25 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs @@ -1,3 +1,4 @@ +using System; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; @@ -7,10 +8,10 @@ namespace StellaOps.Feedser.Storage.Mongo.Dtos; public sealed class DtoDocument { [BsonId] - public Guid Id { get; set; } + public string Id { get; set; } = string.Empty; [BsonElement("documentId")] - public Guid DocumentId { get; set; } + public string DocumentId { get; set; } = string.Empty; [BsonElement("sourceName")] public string SourceName { get; set; } = string.Empty; @@ -30,8 +31,8 @@ internal static class DtoDocumentExtensions public static DtoDocument FromRecord(DtoRecord record) => new() { - Id = record.Id, - DocumentId = record.DocumentId, + Id = record.Id.ToString(), + DocumentId = record.DocumentId.ToString(), SourceName = record.SourceName, SchemaVersion = record.SchemaVersion, Payload = record.Payload ?? new BsonDocument(), @@ -40,8 +41,8 @@ internal static class DtoDocumentExtensions public static DtoRecord ToRecord(this DtoDocument document) => new( - document.Id, - document.DocumentId, + Guid.Parse(document.Id), + Guid.Parse(document.DocumentId), document.SourceName, document.SchemaVersion, document.Payload, diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs index 1d7c02ff..3547c34c 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs @@ -20,7 +20,8 @@ public sealed class DtoStore : IDtoStore ArgumentNullException.ThrowIfNull(record); var document = DtoDocumentExtensions.FromRecord(record); - var filter = Builders.Filter.Eq(x => x.DocumentId, record.DocumentId) + var documentId = record.DocumentId.ToString(); + var filter = Builders.Filter.Eq(x => x.DocumentId, documentId) & Builders.Filter.Eq(x => x.SourceName, record.SourceName); var options = new FindOneAndReplaceOptions @@ -36,7 +37,8 @@ public sealed class DtoStore : IDtoStore public async Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken) { - var document = await _collection.Find(x => x.DocumentId == documentId) + var documentIdValue = documentId.ToString(); + var document = await _collection.Find(x => x.DocumentId == documentIdValue) .FirstOrDefaultAsync(cancellationToken) .ConfigureAwait(false); return document?.ToRecord(); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs index eb83b1cd..a4a696d0 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; +using System.Linq; using MongoDB.Bson.Serialization.Attributes; namespace StellaOps.Feedser.Storage.Mongo.Exporting; @@ -31,6 +33,21 @@ public sealed class ExportStateDocument [BsonElement("updatedAt")] public DateTime UpdatedAt { get; set; } + + [BsonElement("files")] + public List? Files { get; set; } +} + +public sealed class ExportStateFileDocument +{ + [BsonElement("path")] + public string Path { get; set; } = string.Empty; + + [BsonElement("length")] + public long Length { get; set; } + + [BsonElement("digest")] + public string Digest { get; set; } = string.Empty; } internal static class ExportStateDocumentExtensions @@ -47,6 +64,12 @@ internal static class ExportStateDocumentExtensions TargetRepository = record.TargetRepository, ExporterVersion = record.ExporterVersion, UpdatedAt = record.UpdatedAt.UtcDateTime, + Files = record.Files.Select(static file => new ExportStateFileDocument + { + Path = file.Path, + Length = file.Length, + Digest = file.Digest, + }).ToList(), }; public static ExportStateRecord ToRecord(this ExportStateDocument document) @@ -59,5 +82,9 @@ internal static class ExportStateDocumentExtensions document.ExportCursor, document.TargetRepository, document.ExporterVersion, - DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc)); + DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc), + (document.Files ?? new List()) + .Where(static entry => !string.IsNullOrWhiteSpace(entry.Path)) + .Select(static entry => new ExportFileRecord(entry.Path, entry.Length, entry.Digest)) + .ToArray()); } diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs index f2bff595..f3d36f49 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -32,12 +33,14 @@ public sealed class ExportStateManager string? targetRepository, string exporterVersion, bool resetBaseline, + IReadOnlyList manifest, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrEmpty(exporterId); ArgumentException.ThrowIfNullOrEmpty(exportId); ArgumentException.ThrowIfNullOrEmpty(exportDigest); ArgumentException.ThrowIfNullOrEmpty(exporterVersion); + manifest ??= Array.Empty(); var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false); var now = _timeProvider.GetUtcNow(); @@ -55,7 +58,8 @@ public sealed class ExportStateManager ExportCursor: cursor ?? exportDigest, TargetRepository: resolvedRepository, ExporterVersion: exporterVersion, - UpdatedAt: now), + UpdatedAt: now, + Files: manifest), cancellationToken).ConfigureAwait(false); } @@ -81,6 +85,7 @@ public sealed class ExportStateManager TargetRepository = resolvedRepo, ExporterVersion = exporterVersion, UpdatedAt = now, + Files = manifest, } : existing with { @@ -90,6 +95,7 @@ public sealed class ExportStateManager TargetRepository = resolvedRepo, ExporterVersion = exporterVersion, UpdatedAt = now, + Files = manifest, }; return await _store.UpsertAsync(updatedRecord, cancellationToken).ConfigureAwait(false); @@ -100,11 +106,13 @@ public sealed class ExportStateManager string deltaDigest, string? cursor, string exporterVersion, + IReadOnlyList manifest, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrEmpty(exporterId); ArgumentException.ThrowIfNullOrEmpty(deltaDigest); ArgumentException.ThrowIfNullOrEmpty(exporterVersion); + manifest ??= Array.Empty(); var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false); if (existing is null) @@ -119,6 +127,7 @@ public sealed class ExportStateManager ExportCursor = cursor ?? existing.ExportCursor, ExporterVersion = exporterVersion, UpdatedAt = now, + Files = manifest, }; return await _store.UpsertAsync(record, cancellationToken).ConfigureAwait(false); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs index a5597e4e..0ab6ecf0 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs @@ -9,4 +9,7 @@ public sealed record ExportStateRecord( string? ExportCursor, string? TargetRepository, string? ExporterVersion, - DateTimeOffset UpdatedAt); + DateTimeOffset UpdatedAt, + IReadOnlyList Files); + +public sealed record ExportFileRecord(string Path, long Length, string Digest); diff --git a/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs index 3cf6aa75..0e3fc4a4 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Text.Json; @@ -11,8 +12,7 @@ namespace StellaOps.Feedser.Storage.Mongo; public sealed class JobRunDocument { [BsonId] - [BsonGuidRepresentation(GuidRepresentation.Standard)] - public Guid Id { get; set; } + public string Id { get; set; } = string.Empty; [BsonElement("kind")] public string Kind { get; set; } = string.Empty; @@ -60,7 +60,7 @@ internal static class JobRunDocumentExtensions { return new JobRunDocument { - Id = id, + Id = id.ToString(), Kind = request.Kind, Status = JobRunStatus.Pending.ToString(), Trigger = request.Trigger, @@ -79,7 +79,7 @@ internal static class JobRunDocumentExtensions var parameters = document.Parameters?.ToDictionary() ?? new Dictionary(); return new JobRunSnapshot( - document.Id, + Guid.Parse(document.Id), document.Kind, Enum.Parse(document.Status, ignoreCase: true), DateTime.SpecifyKind(document.CreatedAt, DateTimeKind.Utc), diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs index ed6466f9..964c1e8b 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs @@ -1,3 +1,7 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; @@ -6,7 +10,7 @@ namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; public sealed class MergeEventDocument { [BsonId] - public Guid Id { get; set; } + public string Id { get; set; } = string.Empty; [BsonElement("advisoryKey")] public string AdvisoryKey { get; set; } = string.Empty; @@ -21,7 +25,7 @@ public sealed class MergeEventDocument public DateTime MergedAt { get; set; } [BsonElement("inputDocuments")] - public List InputDocuments { get; set; } = new(); + public List InputDocuments { get; set; } = new(); } internal static class MergeEventDocumentExtensions @@ -29,20 +33,20 @@ internal static class MergeEventDocumentExtensions public static MergeEventDocument FromRecord(MergeEventRecord record) => new() { - Id = record.Id, + Id = record.Id.ToString(), AdvisoryKey = record.AdvisoryKey, BeforeHash = record.BeforeHash, AfterHash = record.AfterHash, MergedAt = record.MergedAt.UtcDateTime, - InputDocuments = record.InputDocumentIds.ToList(), + InputDocuments = record.InputDocumentIds.Select(static id => id.ToString()).ToList(), }; public static MergeEventRecord ToRecord(this MergeEventDocument document) => new( - document.Id, + Guid.Parse(document.Id), document.AdvisoryKey, document.BeforeHash, document.AfterHash, DateTime.SpecifyKind(document.MergedAt, DateTimeKind.Utc), - document.InputDocuments); + document.InputDocuments.Select(static value => Guid.Parse(value)).ToList()); } diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs index 9ea86565..6644190f 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs @@ -269,20 +269,22 @@ public sealed class MongoBootstrapper return collection.Indexes.CreateManyAsync(indexes, cancellationToken); } - private Task EnsurePsirtFlagIndexesAsync(CancellationToken cancellationToken) + private async Task EnsurePsirtFlagIndexesAsync(CancellationToken cancellationToken) { var collection = _database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); - var indexes = new List> + try { - new( - Builders.IndexKeys.Ascending("advisoryKey"), - new CreateIndexOptions { Name = "psirt_advisoryKey_unique", Unique = true }), - new( - Builders.IndexKeys.Ascending("vendor"), - new CreateIndexOptions { Name = "psirt_vendor" }), - }; + await collection.Indexes.DropOneAsync("psirt_advisoryKey_unique", cancellationToken).ConfigureAwait(false); + } + catch (MongoCommandException ex) when (ex.CodeName == "IndexNotFound") + { + } - return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + var index = new CreateIndexModel( + Builders.IndexKeys.Ascending("vendor"), + new CreateIndexOptions { Name = "psirt_vendor" }); + + await collection.Indexes.CreateOneAsync(index, cancellationToken: cancellationToken).ConfigureAwait(false); } private Task EnsureChangeHistoryIndexesAsync(CancellationToken cancellationToken) diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs index 7b37cfb9..eca72e27 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs @@ -36,7 +36,8 @@ public sealed class MongoJobStore : IJobStore public async Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) { - var filter = Builders.Filter.Eq(x => x.Id, runId) + var runIdValue = runId.ToString(); + var filter = Builders.Filter.Eq(x => x.Id, runIdValue) & Builders.Filter.Eq(x => x.Status, PendingStatus); var update = Builders.Update @@ -63,7 +64,8 @@ public sealed class MongoJobStore : IJobStore public async Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) { - var filter = Builders.Filter.Eq(x => x.Id, runId) + var runIdValue = runId.ToString(); + var filter = Builders.Filter.Eq(x => x.Id, runIdValue) & Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); var update = Builders.Update @@ -91,7 +93,7 @@ public sealed class MongoJobStore : IJobStore public async Task FindAsync(Guid runId, CancellationToken cancellationToken) { - var cursor = await _collection.FindAsync(x => x.Id == runId, cancellationToken: cancellationToken).ConfigureAwait(false); + var cursor = await _collection.FindAsync(x => x.Id == runId.ToString(), cancellationToken: cancellationToken).ConfigureAwait(false); var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); return document?.ToSnapshot(); } diff --git a/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs index 3ec0f89e..94008ca9 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Options; using MongoDB.Driver; using StellaOps.Feedser.Core.Jobs; using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Aliases; using StellaOps.Feedser.Storage.Mongo.ChangeHistory; using StellaOps.Feedser.Storage.Mongo.Documents; using StellaOps.Feedser.Storage.Mongo.Dtos; @@ -58,6 +59,7 @@ public static class ServiceCollectionExtensions services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/StellaOps.Feedser.Storage.Mongo/TASKS.md b/src/StellaOps.Feedser.Storage.Mongo/TASKS.md index f40a1745..455ff4fb 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/TASKS.md +++ b/src/StellaOps.Feedser.Storage.Mongo/TASKS.md @@ -13,3 +13,4 @@ |Migration playbook for schema/index changes|BE-Storage|Storage.Mongo|DONE – `MongoMigrationRunner` executes `IMongoMigration` steps recorded in `schema_migrations`; see `MIGRATIONS.md`.| |Raw document retention/TTL strategy|BE-Storage|Storage.Mongo|DONE – retention options flow into `RawDocumentRetentionService` and TTL migrations for `document`/GridFS indexes.| |Persist last failure reason in SourceState|BE-Storage|Storage.Mongo|DONE – `MongoSourceStateRepository.MarkFailureAsync` stores `lastFailureReason` with length guard + reset on success.| +|AdvisoryStore range primitives deserialization|BE-Storage|Models|DONE – BSON helpers handle `RangePrimitives`; regression test covers SemVer/NEVRA/EVR envelopes persisted through Mongo.| diff --git a/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs b/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs index d97e42ed..6c4b3a69 100644 --- a/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs +++ b/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs @@ -14,11 +14,7 @@ public sealed class MongoIntegrationFixture : IAsyncLifetime public Task InitializeAsync() { Runner = MongoDbRunner.Start(singleNodeReplSet: true); - var settings = MongoClientSettings.FromConnectionString(Runner.ConnectionString); -#pragma warning disable CS0618 - settings.GuidRepresentation = GuidRepresentation.Standard; -#pragma warning restore CS0618 - Client = new MongoClient(settings); + Client = new MongoClient(Runner.ConnectionString); Database = Client.GetDatabase($"feedser-tests-{Guid.NewGuid():N}"); return Task.CompletedTask; } diff --git a/src/StellaOps.Feedser.WebService/AGENTS.md b/src/StellaOps.Feedser.WebService/AGENTS.md index 20ba4c4e..de375689 100644 --- a/src/StellaOps.Feedser.WebService/AGENTS.md +++ b/src/StellaOps.Feedser.WebService/AGENTS.md @@ -6,8 +6,7 @@ Minimal API host wiring configuration, storage, plugin routines, and job endpoin - Mongo: MongoUrl from options.Storage.Dsn; IMongoClient/IMongoDatabase singletons; default database name fallback (options -> URL -> "feedser"). - Services: AddMongoStorage(); AddSourceHttpClients(); RegisterPluginRoutines(configuration, PluginHostOptions). - Bootstrap: MongoBootstrapper.InitializeAsync on startup. -- Endpoints: - - GET / -> "Hello World!" (placeholder). +- Endpoints (configuration & job control only; root path intentionally unbound): - GET /health -> {status:"healthy"} after options validation binds. - GET /ready -> MongoDB ping; 503 on MongoException/Timeout. - GET /jobs?kind=&limit= -> recent runs. @@ -33,4 +32,3 @@ Out: business logic of jobs, HTML UI, authn/z (future). - Author and review coverage in `../StellaOps.Feedser.WebService.Tests`. - Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - diff --git a/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs b/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs index 3b1bfd39..7faa07d1 100644 --- a/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs +++ b/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs @@ -4,6 +4,7 @@ using System.Globalization; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Merge.Jobs; namespace StellaOps.Feedser.WebService.Extensions; @@ -52,7 +53,8 @@ internal static class JobRegistrationExtensions new("source:vndr-oracle:map", "StellaOps.Feedser.Source.Vndr.Oracle.OracleMapJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), new("export:json", "StellaOps.Feedser.Exporter.Json.JsonExportJob", "StellaOps.Feedser.Exporter.Json", TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(5)), - new("export:trivy-db", "StellaOps.Feedser.Exporter.TrivyDb.TrivyDbExportJob", "StellaOps.Feedser.Exporter.TrivyDb", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(10)) + new("export:trivy-db", "StellaOps.Feedser.Exporter.TrivyDb.TrivyDbExportJob", "StellaOps.Feedser.Exporter.TrivyDb", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(10)), + new(MergeJobKinds.Reconcile, "StellaOps.Feedser.Merge.Jobs.MergeReconcileJob", "StellaOps.Feedser.Merge", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)) }; public static IServiceCollection AddBuiltInFeedserJobs(this IServiceCollection services) diff --git a/src/StellaOps.Feedser.WebService/Program.cs b/src/StellaOps.Feedser.WebService/Program.cs index 7a47bf56..e0dbdb4d 100644 --- a/src/StellaOps.Feedser.WebService/Program.cs +++ b/src/StellaOps.Feedser.WebService/Program.cs @@ -16,44 +16,50 @@ using StellaOps.Feedser.Core.Jobs; using StellaOps.Feedser.Storage.Mongo; using StellaOps.Feedser.WebService.Diagnostics; using Serilog; +using StellaOps.Feedser.Merge; +using StellaOps.Feedser.Merge.Services; using StellaOps.Feedser.WebService.Extensions; using StellaOps.Feedser.WebService.Jobs; using StellaOps.Feedser.WebService.Options; using Serilog.Events; using StellaOps.Plugin.DependencyInjection; using StellaOps.Plugin.Hosting; +using StellaOps.Configuration; var builder = WebApplication.CreateBuilder(args); -builder.Configuration - .AddJsonFile("appsettings.json", optional: true, reloadOnChange: false) - .AddFeedserYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/feedser.yaml")) - .AddEnvironmentVariables(prefix: "FEEDSER_"); +builder.Configuration.AddStellaOpsDefaults(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "FEEDSER_"; + options.ConfigureBuilder = configurationBuilder => + { + configurationBuilder.AddFeedserYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/feedser.yaml")); + }; +}); +var feedserOptions = builder.Configuration.BindOptions(postConfigure: (opts, _) => FeedserOptionsValidator.Validate(opts)); builder.Services.AddOptions() .Bind(builder.Configuration) .PostConfigure(FeedserOptionsValidator.Validate) .ValidateOnStart(); -var optionsSnapshot = new FeedserOptions(); -builder.Configuration.Bind(optionsSnapshot); -FeedserOptionsValidator.Validate(optionsSnapshot); - -builder.ConfigureFeedserTelemetry(optionsSnapshot); +builder.ConfigureFeedserTelemetry(feedserOptions); builder.Services.AddMongoStorage(storageOptions => { - storageOptions.ConnectionString = optionsSnapshot.Storage.Dsn; - storageOptions.DatabaseName = optionsSnapshot.Storage.Database; - storageOptions.CommandTimeout = TimeSpan.FromSeconds(optionsSnapshot.Storage.CommandTimeoutSeconds); + storageOptions.ConnectionString = feedserOptions.Storage.Dsn; + storageOptions.DatabaseName = feedserOptions.Storage.Database; + storageOptions.CommandTimeout = TimeSpan.FromSeconds(feedserOptions.Storage.CommandTimeoutSeconds); }); +builder.Services.AddMergeModule(builder.Configuration); builder.Services.AddJobScheduler(); builder.Services.AddBuiltInFeedserJobs(); builder.Services.AddSingleton(sp => new ServiceStatus(sp.GetRequiredService())); -var pluginHostOptions = BuildPluginOptions(optionsSnapshot, builder.Environment.ContentRootPath); +var pluginHostOptions = BuildPluginOptions(feedserOptions, builder.Environment.ContentRootPath); builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions); builder.Services.AddEndpointsApiExplorer(); @@ -153,8 +159,6 @@ void ApplyNoCache(HttpResponse response) await InitializeMongoAsync(app); -app.MapGet("/", () => Results.Ok(new { message = "StellaOps Feedser" })); - app.MapGet("/health", (IOptions opts, ServiceStatus status, HttpContext context) => { ApplyNoCache(context.Response); @@ -242,6 +246,46 @@ app.MapGet("/ready", async (IMongoDatabase database, ServiceStatus status, HttpC } }); +app.MapGet("/diagnostics/aliases/{seed}", async (string seed, AliasGraphResolver resolver, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + if (string.IsNullOrWhiteSpace(seed)) + { + return Problem(context, "Seed advisory key is required.", StatusCodes.Status400BadRequest, ProblemTypes.Validation); + } + + var component = await resolver.BuildComponentAsync(seed, cancellationToken).ConfigureAwait(false); + + var aliases = component.AliasMap.ToDictionary( + static kvp => kvp.Key, + static kvp => kvp.Value + .Select(record => new + { + record.Scheme, + record.Value, + UpdatedAt = record.UpdatedAt + }) + .ToArray()); + + var response = new + { + Seed = component.SeedAdvisoryKey, + Advisories = component.AdvisoryKeys, + Collisions = component.Collisions + .Select(collision => new + { + collision.Scheme, + collision.Value, + AdvisoryKeys = collision.AdvisoryKeys + }) + .ToArray(), + Aliases = aliases + }; + + return JsonResult(response); +}); + app.MapGet("/jobs", async (string? kind, int? limit, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => { ApplyNoCache(context.Response); diff --git a/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj b/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj index cc8f5866..bfcba117 100644 --- a/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj +++ b/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj @@ -25,7 +25,9 @@ + + diff --git a/src/StellaOps.Feedser.sln b/src/StellaOps.Feedser.sln index c542ef93..a57e7912 100644 --- a/src/StellaOps.Feedser.sln +++ b/src/StellaOps.Feedser.sln @@ -123,6 +123,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Os EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Testing", "StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj", "{EAE910FC-188C-41C3-822A-623964CABE48}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian.Tests", "StellaOps.Feedser.Source.Distro.Debian.Tests\StellaOps.Feedser.Source.Distro.Debian.Tests.csproj", "{BBA5C780-6348-427D-9600-726EAA8963B3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", "StellaOps.Configuration\StellaOps.Configuration.csproj", "{5F44A429-816A-4560-A5AA-61CD23FD8A19}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli", "StellaOps.Cli\StellaOps.Cli.csproj", "{20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli.Tests", "StellaOps.Cli.Tests\StellaOps.Cli.Tests.csproj", "{544DBB82-4639-4856-A5F2-76828F7A8396}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -853,6 +861,54 @@ Global {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.Build.0 = Release|Any CPU {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.ActiveCfg = Release|Any CPU {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x64.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x86.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|Any CPU.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x64.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x64.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x86.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x86.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x64.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x64.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x86.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x86.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|Any CPU.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x64.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x64.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x86.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x86.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x64.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x64.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x86.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x86.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|Any CPU.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x64.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x64.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x86.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x86.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|Any CPU.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x64.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x64.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x86.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x86.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|Any CPU.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|Any CPU.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x64.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x64.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x86.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE