From 6cbfd47ecdc4565314149e850cf7c265926f812c Mon Sep 17 00:00:00 2001 From: master Date: Tue, 7 Oct 2025 10:14:21 +0300 Subject: [PATCH] Initial commit (history squashed) --- .../_deprecated-feedser-ci.yml.disabled | 29 + .../_deprecated-feedser-tests.yml.disabled | 87 + .gitea/workflows/build-test-deploy.yml | 297 ++++ .gitea/workflows/docs.yml | 70 + .gitea/workflows/promote.yml | 206 +++ .gitignore | 21 + AGENTS.md | 125 ++ LICENSE | 235 +++ README.md | 2 + docs/01_WHAT_IS_IT.md | 77 + docs/02_WHY.md | 121 ++ docs/03_QUICKSTART.md | 156 ++ docs/03_VISION.md | 99 ++ docs/04_FEATURE_MATRIX.md | 34 + docs/05_ROADMAP.md | 6 + docs/05_SYSTEM_REQUIREMENTS_SPEC.md | 204 +++ docs/07_HIGH_LEVEL_ARCHITECTURE.md | 388 ++++ docs/08_MODULE_SPECIFICATIONS.md | 201 +++ docs/09_API_CLI_REFERENCE.md | 329 ++++ docs/10_OFFLINE_KIT.md | 139 ++ docs/10_PLUGIN_SDK_GUIDE.md | 194 ++ docs/11_DATA_SCHEMAS.md | 196 +++ docs/11_GOVERNANCE.md | 93 + docs/12_CODE_OF_CONDUCT.md | 88 + docs/12_PERFORMANCE_WORKBOOK.md | 167 ++ docs/13_RELEASE_ENGINEERING_PLAYBOOK.md | 209 +++ docs/13_SECURITY_POLICY.md | 101 ++ docs/14_GLOSSARY_OF_TERMS.md | 112 ++ docs/15_UI_GUIDE.md | 234 +++ docs/17_SECURITY_HARDENING_GUIDE.md | 186 ++ docs/18_CODING_STANDARDS.md | 169 ++ docs/19_TEST_SUITE_OVERVIEW.md | 91 + docs/21_INSTALL_GUIDE.md | 131 ++ docs/23_FAQ_MATRIX.md | 61 + docs/24_OFFLINE_KIT.md | 94 + docs/29_LEGAL_FAQ_QUOTA.md | 84 + docs/30_QUOTA_ENFORCEMENT_FLOW1.md | 93 + docs/33_333_QUOTA_OVERVIEW.md | 120 ++ docs/40_ARCHITECTURE_OVERVIEW.md | 133 ++ docs/60_POLICY_TEMPLATES.md | 101 ++ docs/ARCHITECTURE_FEEDSER.md | 190 ++ docs/README.md | 67 + docs/_includes/CONSTANTS.md | 18 + docs/ci/20_CI_RECIPES.md | 258 +++ docs/cli/20_REFERENCE.md | 8 + docs/dev/30_PLUGIN_DEV_GUIDE.md | 146 ++ docs/license-jwt-quota.md | 123 ++ global.json | 6 + scripts/render_docs.py | 254 +++ src/Directory.Build.props | 29 + src/Directory.Build.targets | 17 + src/Jobs.cs | 46 + src/OracleConnector.cs | 293 +++ src/OracleConnectorPlugin.cs | 21 + src/OracleDependencyInjectionRoutine.cs | 54 + .../IDependencyInjectionRoutine.cs | 11 + .../StellaOps.DependencyInjection.csproj | 14 + .../JobCoordinatorTests.cs | 483 +++++ .../JobPluginRegistrationExtensionsTests.cs | 61 + .../JobSchedulerBuilderTests.cs | 70 + .../PluginRoutineFixtures.cs | 42 + .../StellaOps.Feedser.Core.Tests.csproj | 10 + src/StellaOps.Feedser.Core/AGENTS.md | 32 + src/StellaOps.Feedser.Core/Jobs/IJob.cs | 6 + .../Jobs/IJobCoordinator.cs | 18 + src/StellaOps.Feedser.Core/Jobs/IJobStore.cs | 20 + .../Jobs/ILeaseStore.cs | 10 + .../Jobs/JobCoordinator.cs | 635 +++++++ .../Jobs/JobDefinition.cs | 12 + .../Jobs/JobDiagnostics.cs | 171 ++ .../Jobs/JobExecutionContext.cs | 42 + src/StellaOps.Feedser.Core/Jobs/JobLease.cs | 9 + .../Jobs/JobPluginRegistrationExtensions.cs | 128 ++ .../Jobs/JobRunCompletion.cs | 6 + .../Jobs/JobRunCreateRequest.cs | 10 + .../Jobs/JobRunSnapshot.cs | 21 + .../Jobs/JobRunStatus.cs | 10 + .../Jobs/JobSchedulerBuilder.cs | 47 + .../Jobs/JobSchedulerHostedService.cs | 165 ++ .../Jobs/JobSchedulerOptions.cs | 12 + .../Jobs/JobTriggerResult.cs | 40 + .../Jobs/ServiceCollectionExtensions.cs | 27 + .../StellaOps.Feedser.Core.csproj | 19 + src/StellaOps.Feedser.Core/TASKS.md | 14 + .../JsonExportSnapshotBuilderTests.cs | 213 +++ ...ExporterDependencyInjectionRoutineTests.cs | 83 + .../JsonExporterParitySmokeTests.cs | 182 ++ .../JsonFeedExporterTests.cs | 265 +++ ...ellaOps.Feedser.Exporter.Json.Tests.csproj | 13 + .../VulnListJsonExportPathResolverTests.cs | 148 ++ src/StellaOps.Feedser.Exporter.Json/AGENTS.md | 28 + .../ExportDigestCalculator.cs | 52 + .../ExporterVersion.cs | 28 + .../IJsonExportPathResolver.cs | 12 + .../JsonExportFile.cs | 37 + .../JsonExportJob.cs | 30 + .../JsonExportManifestWriter.cs | 66 + .../JsonExportOptions.cs | 34 + .../JsonExportResult.cs | 46 + .../JsonExportSnapshotBuilder.cs | 239 +++ .../JsonExporterDependencyInjectionRoutine.cs | 59 + .../JsonExporterPlugin.cs | 23 + .../JsonFeedExporter.cs | 150 ++ .../StellaOps.Feedser.Exporter.Json.csproj | 22 + src/StellaOps.Feedser.Exporter.Json/TASKS.md | 11 + .../VulnListJsonExportPathResolver.cs | 455 +++++ ...aOps.Feedser.Exporter.TrivyDb.Tests.csproj | 13 + .../TrivyDbExportPlannerTests.cs | 66 + .../TrivyDbFeedExporterTests.cs | 589 +++++++ .../TrivyDbOciWriterTests.cs | 113 ++ .../TrivyDbPackageBuilderTests.cs | 93 + .../AGENTS.md | 29 + .../ITrivyDbBuilder.cs | 15 + .../ITrivyDbOrasPusher.cs | 9 + .../OciDescriptor.cs | 10 + .../OciIndex.cs | 8 + .../OciManifest.cs | 10 + .../StellaOps.Feedser.Exporter.TrivyDb.csproj | 22 + .../TASKS.md | 13 + .../TrivyConfigDocument.cs | 11 + .../TrivyDbBlob.cs | 78 + .../TrivyDbBoltBuilder.cs | 376 ++++ .../TrivyDbBuilderResult.cs | 10 + .../TrivyDbExportJob.cs | 30 + .../TrivyDbExportMode.cs | 8 + .../TrivyDbExportOptions.cs | 80 + .../TrivyDbExportPlan.cs | 7 + .../TrivyDbExportPlanner.cs | 33 + ...ivyDbExporterDependencyInjectionRoutine.cs | 64 + .../TrivyDbExporterPlugin.cs | 23 + .../TrivyDbFeedExporter.cs | 384 ++++ .../TrivyDbMediaTypes.cs | 9 + .../TrivyDbOciWriteResult.cs | 8 + .../TrivyDbOciWriter.cs | 180 ++ .../TrivyDbOrasPusher.cs | 209 +++ .../TrivyDbPackage.cs | 9 + .../TrivyDbPackageBuilder.cs | 116 ++ .../TrivyDbPackageRequest.cs | 11 + .../AdvisoryPrecedenceMergerTests.cs | 262 +++ .../AffectedPackagePrecedenceResolverTests.cs | 88 + .../CanonicalHashCalculatorTests.cs | 86 + .../DebianEvrComparerTests.cs | 84 + .../MergeEventWriterTests.cs | 85 + .../MergePrecedenceIntegrationTests.cs | 211 +++ .../MetricCollector.cs | 56 + .../NevraComparerTests.cs | 108 ++ .../SemanticVersionRangeResolverTests.cs | 67 + .../StellaOps.Feedser.Merge.Tests.csproj | 13 + .../TestLogger.cs | 52 + src/StellaOps.Feedser.Merge/AGENTS.md | 33 + src/StellaOps.Feedser.Merge/Class1.cs | 1 + .../Comparers/DebianEvr.cs | 232 +++ .../Comparers/Nevra.cs | 264 +++ .../Comparers/SemanticVersionRangeResolver.cs | 73 + .../Options/AdvisoryPrecedenceOptions.cs | 15 + .../Options/AdvisoryPrecedenceTable.cs | 35 + .../Services/AdvisoryPrecedenceMerger.cs | 357 ++++ .../AffectedPackagePrecedenceResolver.cs | 114 ++ .../Services/CanonicalHashCalculator.cs | 25 + .../Services/MergeEventWriter.cs | 70 + .../StellaOps.Feedser.Merge.csproj | 16 + src/StellaOps.Feedser.Merge/TASKS.md | 13 + .../AdvisoryTests.cs | 62 + .../AffectedPackageStatusTests.cs | 28 + .../AliasSchemeRegistryTests.cs | 52 + .../CanonicalExampleFactory.cs | 195 ++ .../CanonicalExamplesTests.cs | 57 + .../CanonicalJsonSerializerTests.cs | 65 + .../SeverityNormalizationTests.cs | 28 + .../StellaOps.Feedser.Models.Tests.csproj | 10 + src/StellaOps.Feedser.Models/AGENTS.md | 30 + src/StellaOps.Feedser.Models/Advisory.cs | 145 ++ .../AdvisoryProvenance.cs | 28 + .../AdvisoryReference.cs | 36 + .../AffectedPackage.cs | 87 + .../AffectedPackageStatus.cs | 46 + .../AffectedPackageStatusCatalog.cs | 55 + .../AffectedVersionRange.cs | 145 ++ .../AliasSchemeRegistry.cs | 166 ++ src/StellaOps.Feedser.Models/AliasSchemes.cs | 31 + .../BACKWARD_COMPATIBILITY.md | 41 + .../CANONICAL_RECORDS.md | 128 ++ .../CanonicalJsonSerializer.cs | 91 + src/StellaOps.Feedser.Models/CvssMetric.cs | 31 + .../PROVENANCE_GUIDELINES.md | 12 + .../SeverityNormalization.cs | 68 + .../SnapshotSerializer.cs | 27 + .../StellaOps.Feedser.Models.csproj | 9 + src/StellaOps.Feedser.Models/TASKS.md | 18 + src/StellaOps.Feedser.Models/Validation.cs | 57 + .../CpeNormalizerTests.cs | 70 + .../CvssMetricNormalizerTests.cs | 52 + .../DebianEvrParserTests.cs | 31 + .../DescriptionNormalizerTests.cs | 44 + .../NevraParserTests.cs | 64 + .../PackageUrlNormalizerTests.cs | 44 + ...ellaOps.Feedser.Normalization.Tests.csproj | 11 + .../AssemblyInfo.cs | 8 + .../Cvss/CvssMetricNormalizer.cs | 529 ++++++ .../Distro/DebianEvr.cs | 127 ++ .../Distro/Nevra.cs | 192 ++ .../Identifiers/Cpe23.cs | 352 ++++ .../Identifiers/IdentifierNormalizer.cs | 32 + .../Identifiers/PackageUrl.cs | 299 ++++ .../StellaOps.Feedser.Normalization.csproj | 18 + src/StellaOps.Feedser.Normalization/TASKS.md | 8 + .../Text/DescriptionNormalizer.cs | 118 ++ src/StellaOps.Feedser.Source.Acsc/Class1.cs | 29 + .../StellaOps.Feedser.Source.Acsc.csproj | 16 + src/StellaOps.Feedser.Source.Cccs/Class1.cs | 29 + .../StellaOps.Feedser.Source.Cccs.csproj | 16 + .../Class1.cs | 29 + .../StellaOps.Feedser.Source.CertBund.csproj | 16 + src/StellaOps.Feedser.Source.CertCc/Class1.cs | 29 + .../StellaOps.Feedser.Source.CertCc.csproj | 16 + .../CertFr/CertFrConnectorTests.cs | 305 ++++ .../Fixtures/certfr-advisories.snapshot.json | 112 ++ .../Fixtures/certfr-detail-AV-2024-001.html | 8 + .../Fixtures/certfr-detail-AV-2024-002.html | 11 + .../CertFr/Fixtures/certfr-feed.xml | 22 + ...ellaOps.Feedser.Source.CertFr.Tests.csproj | 16 + src/StellaOps.Feedser.Source.CertFr/AGENTS.md | 28 + .../CertFrConnector.cs | 337 ++++ .../CertFrConnectorPlugin.cs | 21 + .../CertFrDependencyInjectionRoutine.cs | 54 + .../CertFrServiceCollectionExtensions.cs | 36 + .../Configuration/CertFrOptions.cs | 46 + .../Internal/CertFrCursor.cs | 88 + .../Internal/CertFrDocumentMetadata.cs | 77 + .../Internal/CertFrDto.cs | 14 + .../Internal/CertFrFeedClient.cs | 109 ++ .../Internal/CertFrFeedItem.cs | 10 + .../Internal/CertFrMapper.cs | 66 + .../Internal/CertFrParser.cs | 80 + src/StellaOps.Feedser.Source.CertFr/Jobs.cs | 46 + .../StellaOps.Feedser.Source.CertFr.csproj | 13 + src/StellaOps.Feedser.Source.CertFr/TASKS.md | 11 + .../CertIn/CertInConnectorTests.cs | 340 ++++ .../CertIn/Fixtures/alerts-page1.json | 9 + .../Fixtures/detail-CIAD-2024-0005.html | 17 + .../CertIn/Fixtures/expected-advisory.json | 97 + ...ellaOps.Feedser.Source.CertIn.Tests.csproj | 16 + src/StellaOps.Feedser.Source.CertIn/AGENTS.md | 29 + .../CertInConnector.cs | 440 +++++ .../CertInConnectorPlugin.cs | 19 + .../CertInDependencyInjectionRoutine.cs | 54 + .../CertInServiceCollectionExtensions.cs | 37 + .../Configuration/CertInOptions.cs | 68 + .../Internal/CertInAdvisoryDto.cs | 16 + .../Internal/CertInClient.cs | 129 ++ .../Internal/CertInCursor.cs | 88 + .../Internal/CertInDetailParser.cs | 187 ++ .../Internal/CertInListingItem.cs | 10 + src/StellaOps.Feedser.Source.CertIn/Jobs.cs | 46 + .../StellaOps.Feedser.Source.CertIn.csproj | 16 + src/StellaOps.Feedser.Source.CertIn/TASKS.md | 10 + .../Common/CannedHttpMessageHandlerTests.cs | 37 + .../Common/HtmlContentSanitizerTests.cs | 31 + .../Common/PackageCoordinateHelperTests.cs | 41 + .../Common/PdfTextExtractorTests.cs | 21 + .../Common/SourceFetchServiceTests.cs | 36 + .../Common/TimeWindowCursorPlannerTests.cs | 87 + .../Common/UrlNormalizerTests.cs | 24 + .../Json/JsonSchemaValidatorTests.cs | 51 + ...ellaOps.Feedser.Source.Common.Tests.csproj | 10 + .../Xml/XmlSchemaValidatorTests.cs | 58 + src/StellaOps.Feedser.Source.Common/AGENTS.md | 32 + .../Cursors/PaginationPlanner.cs | 29 + .../Cursors/TimeWindowCursorOptions.cs | 43 + .../Cursors/TimeWindowCursorPlanner.cs | 50 + .../Cursors/TimeWindowCursorState.cs | 84 + .../DocumentStatuses.cs | 27 + .../Fetch/CryptoJitterSource.cs | 43 + .../Fetch/IJitterSource.cs | 9 + .../Fetch/RawDocumentStorage.cs | 90 + .../Fetch/SourceFetchContentResult.cs | 58 + .../Fetch/SourceFetchRequest.cs | 24 + .../Fetch/SourceFetchResult.cs | 34 + .../Fetch/SourceFetchService.cs | 313 ++++ .../Fetch/SourceRetryPolicy.cs | 79 + .../Html/HtmlContentSanitizer.cs | 168 ++ .../Http/AllowlistedHttpMessageHandler.cs | 36 + .../Http/ServiceCollectionExtensions.cs | 76 + .../Http/SourceHttpClientOptions.cs | 80 + .../Json/IJsonSchemaValidator.cs | 9 + .../Json/JsonSchemaValidationError.cs | 7 + .../Json/JsonSchemaValidationException.cs | 15 + .../Json/JsonSchemaValidator.cs | 92 + .../Packages/PackageCoordinateHelper.cs | 142 ++ .../Pdf/PdfTextExtractor.cs | 103 ++ .../Properties/AssemblyInfo.cs | 3 + .../StellaOps.Feedser.Source.Common.csproj | 21 + src/StellaOps.Feedser.Source.Common/TASKS.md | 16 + .../Telemetry/SourceDiagnostics.cs | 107 ++ .../Testing/CannedHttpMessageHandler.cs | 210 +++ .../Url/UrlNormalizer.cs | 62 + .../Xml/IXmlSchemaValidator.cs | 9 + .../Xml/XmlSchemaValidationError.cs | 3 + .../Xml/XmlSchemaValidationException.cs | 18 + .../Xml/XmlSchemaValidator.cs | 71 + src/StellaOps.Feedser.Source.Cve/Class1.cs | 29 + .../StellaOps.Feedser.Source.Cve.csproj | 16 + .../Class1.cs | 29 + ...llaOps.Feedser.Source.Distro.Debian.csproj | 16 + .../RedHat/Fixtures/csaf-rhsa-2025-0001.json | 95 + .../RedHat/Fixtures/csaf-rhsa-2025-0002.json | 82 + .../RedHat/Fixtures/csaf-rhsa-2025-0003.json | 93 + .../Fixtures/rhsa-2025-0001.snapshot.json | 118 ++ .../RedHat/Fixtures/summary-page1-repeat.json | 8 + .../RedHat/Fixtures/summary-page1.json | 8 + .../RedHat/Fixtures/summary-page2.json | 8 + .../RedHat/Fixtures/summary-page3.json | 8 + .../RedHat/RedHatConnectorHarnessTests.cs | 114 ++ .../RedHat/RedHatConnectorTests.cs | 449 +++++ ....Feedser.Source.Distro.RedHat.Tests.csproj | 16 + .../AGENTS.md | 28 + .../Configuration/RedHatOptions.cs | 97 + .../Internal/Models/RedHatCsafModels.cs | 177 ++ .../Internal/RedHatCursor.cs | 254 +++ .../Internal/RedHatMapper.cs | 718 ++++++++ .../Internal/RedHatSummaryItem.cs | 66 + .../Jobs.cs | 46 + .../RedHatConnector.cs | 432 +++++ .../RedHatConnectorPlugin.cs | 19 + .../RedHatDependencyInjectionRoutine.cs | 54 + .../RedHatServiceCollectionExtensions.cs | 34 + ...llaOps.Feedser.Source.Distro.RedHat.csproj | 15 + .../TASKS.md | 15 + .../Class1.cs | 29 + ...tellaOps.Feedser.Source.Distro.Suse.csproj | 16 + .../Class1.cs | 29 + ...llaOps.Feedser.Source.Distro.Ubuntu.csproj | 16 + src/StellaOps.Feedser.Source.Ghsa/Class1.cs | 29 + .../StellaOps.Feedser.Source.Ghsa.csproj | 16 + .../Class1.cs | 29 + .../StellaOps.Feedser.Source.Ics.Cisa.csproj | 16 + .../Fixtures/detail-acme-controller-2024.html | 18 + .../Kaspersky/Fixtures/expected-advisory.json | 235 +++ .../Kaspersky/Fixtures/feed-page1.xml | 17 + .../Kaspersky/KasperskyConnectorTests.cs | 338 ++++ ....Feedser.Source.Ics.Kaspersky.Tests.csproj | 16 + .../AGENTS.md | 29 + .../Configuration/KasperskyOptions.cs | 53 + .../Internal/KasperskyAdvisoryDto.cs | 14 + .../Internal/KasperskyAdvisoryParser.cs | 172 ++ .../Internal/KasperskyCursor.cs | 207 +++ .../Internal/KasperskyFeedClient.cs | 133 ++ .../Internal/KasperskyFeedItem.cs | 9 + .../Jobs.cs | 46 + .../KasperskyConnector.cs | 445 +++++ .../KasperskyConnectorPlugin.cs | 19 + .../KasperskyDependencyInjectionRoutine.cs | 54 + .../KasperskyServiceCollectionExtensions.cs | 37 + ...llaOps.Feedser.Source.Ics.Kaspersky.csproj | 16 + .../TASKS.md | 10 + .../Jvn/Fixtures/expected-advisory.json | 84 + .../Jvn/Fixtures/jvnrss-window1.xml | 53 + .../Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml | 95 + .../Jvn/JvnConnectorTests.cs | 264 +++ .../StellaOps.Feedser.Source.Jvn.Tests.csproj | 16 + src/StellaOps.Feedser.Source.Jvn/AGENTS.md | 30 + .../Configuration/JvnOptions.cs | 80 + .../Internal/JvnAdvisoryMapper.cs | 347 ++++ .../Internal/JvnConstants.cs | 10 + .../Internal/JvnCursor.cs | 106 ++ .../Internal/JvnDetailDto.cs | 67 + .../Internal/JvnDetailParser.cs | 296 ++++ .../Internal/JvnOverviewItem.cs | 8 + .../Internal/JvnOverviewPage.cs | 7 + .../Internal/JvnSchemaProvider.cs | 167 ++ .../Internal/JvnSchemaValidationException.cs | 16 + .../Internal/MyJvnClient.cs | 240 +++ src/StellaOps.Feedser.Source.Jvn/Jobs.cs | 46 + .../JvnConnector.cs | 321 ++++ .../JvnConnectorPlugin.cs | 19 + .../JvnDependencyInjectionRoutine.cs | 54 + .../JvnServiceCollectionExtensions.cs | 37 + .../Schemas/data_marking.xsd | 91 + .../Schemas/jvnrss_3.2.xsd | 133 ++ .../Schemas/mod_sec_3.0.xsd | 168 ++ .../Schemas/status_3.3.xsd | 574 ++++++ .../Schemas/tlp_marking.xsd | 40 + .../Schemas/vuldef_3.2.xsd | 1564 +++++++++++++++++ .../Schemas/xml.xsd | 287 +++ .../StellaOps.Feedser.Source.Jvn.csproj | 15 + src/StellaOps.Feedser.Source.Jvn/TASKS.md | 13 + src/StellaOps.Feedser.Source.Kev/Class1.cs | 29 + .../StellaOps.Feedser.Source.Kev.csproj | 16 + src/StellaOps.Feedser.Source.Kisa/Class1.cs | 29 + .../StellaOps.Feedser.Source.Kisa.csproj | 16 + .../Nvd/Fixtures/nvd-invalid-schema.json | 6 + .../Nvd/Fixtures/nvd-multipage-1.json | 69 + .../Nvd/Fixtures/nvd-multipage-2.json | 69 + .../Nvd/Fixtures/nvd-multipage-3.json | 38 + .../Nvd/Fixtures/nvd-window-1.json | 85 + .../Nvd/Fixtures/nvd-window-2.json | 45 + .../Nvd/Fixtures/nvd-window-update.json | 51 + .../Nvd/NvdConnectorHarnessTests.cs | 124 ++ .../Nvd/NvdConnectorTests.cs | 607 +++++++ .../StellaOps.Feedser.Source.Nvd.Tests.csproj | 16 + src/StellaOps.Feedser.Source.Nvd/AGENTS.md | 27 + .../Configuration/NvdOptions.cs | 57 + .../Internal/NvdCursor.cs | 64 + .../Internal/NvdDiagnostics.cs | 76 + .../Internal/NvdMapper.cs | 293 +++ .../Internal/NvdSchemaProvider.cs | 25 + .../NvdConnector.cs | 565 ++++++ .../NvdConnectorPlugin.cs | 19 + .../NvdServiceCollectionExtensions.cs | 35 + .../Schemas/nvd-vulnerability.schema.json | 115 ++ .../StellaOps.Feedser.Source.Nvd.csproj | 17 + src/StellaOps.Feedser.Source.Nvd/TASKS.md | 13 + .../Osv/OsvMapperTests.cs | 117 ++ .../StellaOps.Feedser.Source.Osv.Tests.csproj | 13 + src/StellaOps.Feedser.Source.Osv/AGENTS.md | 27 + .../Configuration/OsvOptions.cs | 81 + .../Internal/OsvCursor.cs | 290 +++ .../Internal/OsvMapper.cs | 391 +++++ .../Internal/OsvVulnerabilityDto.cs | 114 ++ src/StellaOps.Feedser.Source.Osv/Jobs.cs | 46 + .../OsvConnector.cs | 497 ++++++ .../OsvConnectorPlugin.cs | 20 + .../OsvDependencyInjectionRoutine.cs | 53 + .../OsvServiceCollectionExtensions.cs | 37 + .../StellaOps.Feedser.Source.Osv.csproj | 20 + src/StellaOps.Feedser.Source.Osv/TASKS.md | 13 + src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs | 29 + .../StellaOps.Feedser.Source.Ru.Bdu.csproj | 16 + .../Class1.cs | 29 + .../StellaOps.Feedser.Source.Ru.Nkcki.csproj | 16 + .../Adobe/AdobeConnectorFetchTests.cs | 351 ++++ .../Fixtures/adobe-advisories.snapshot.json | 108 ++ .../Fixtures/adobe-detail-apsb25-85.html | 10 + .../Fixtures/adobe-detail-apsb25-87.html | 10 + .../Adobe/Fixtures/adobe-index.html | 17 + ...Ops.Feedser.Source.Vndr.Adobe.Tests.csproj | 17 + .../AGENTS.md | 29 + .../AdobeConnector.cs | 503 ++++++ .../AdobeConnectorPlugin.cs | 21 + .../AdobeDiagnostics.cs | 49 + .../AdobeServiceCollectionExtensions.cs | 38 + .../Configuration/AdobeOptions.cs | 50 + .../Internal/AdobeBulletinDto.cs | 50 + .../Internal/AdobeCursor.cs | 168 ++ .../Internal/AdobeDetailParser.cs | 133 ++ .../Internal/AdobeDocumentMetadata.cs | 47 + .../Internal/AdobeIndexEntry.cs | 5 + .../Internal/AdobeIndexParser.cs | 159 ++ .../Internal/AdobeSchemaProvider.cs | 25 + .../Schemas/adobe-bulletin.schema.json | 48 + ...StellaOps.Feedser.Source.Vndr.Adobe.csproj | 25 + .../TASKS.md | 11 + .../Class1.cs | 29 + ...StellaOps.Feedser.Source.Vndr.Apple.csproj | 16 + .../Chromium/ChromiumConnectorTests.cs | 347 ++++ .../Chromium/ChromiumMapperTests.cs | 47 + .../Fixtures/chromium-advisory.snapshot.json | 1 + .../Chromium/Fixtures/chromium-detail.html | 21 + .../Chromium/Fixtures/chromium-feed.xml | 16 + ....Feedser.Source.Vndr.Chromium.Tests.csproj | 18 + .../AGENTS.md | 29 + .../ChromiumConnector.cs | 364 ++++ .../ChromiumConnectorPlugin.cs | 20 + .../ChromiumDiagnostics.cs | 69 + .../ChromiumServiceCollectionExtensions.cs | 37 + .../Configuration/ChromiumOptions.cs | 44 + .../Internal/ChromiumCursor.cs | 143 ++ .../Internal/ChromiumDocumentMetadata.cs | 78 + .../Internal/ChromiumDto.cs | 39 + .../Internal/ChromiumFeedEntry.cs | 24 + .../Internal/ChromiumFeedLoader.cs | 147 ++ .../Internal/ChromiumMapper.cs | 120 ++ .../Internal/ChromiumParser.cs | 282 +++ .../Internal/ChromiumSchemaProvider.cs | 25 + .../Properties/AssemblyInfo.cs | 3 + .../Schemas/chromium-post.schema.json | 97 + ...llaOps.Feedser.Source.Vndr.Chromium.csproj | 32 + .../TASKS.md | 16 + .../Class1.cs | 29 + ...StellaOps.Feedser.Source.Vndr.Cisco.csproj | 16 + .../Class1.cs | 29 + .../StellaOps.Feedser.Source.Vndr.Msrc.csproj | 16 + .../Fixtures/oracle-advisories.snapshot.json | 112 ++ .../Fixtures/oracle-detail-cpuapr2024-01.html | 11 + .../Fixtures/oracle-detail-cpuapr2024-02.html | 8 + .../Oracle/OracleConnectorTests.cs | 158 ++ ...ps.Feedser.Source.Vndr.Oracle.Tests.csproj | 16 + .../AGENTS.md | 28 + .../Configuration/OracleOptions.cs | 32 + .../Internal/OracleCursor.cs | 88 + .../Internal/OracleDocumentMetadata.cs | 56 + .../Internal/OracleDto.cs | 13 + .../Internal/OracleMapper.cs | 76 + .../Internal/OracleParser.cs | 53 + .../OracleConnector.cs | 295 ++++ .../OracleServiceCollectionExtensions.cs | 37 + ...tellaOps.Feedser.Source.Vndr.Oracle.csproj | 17 + .../TASKS.md | 12 + .../VndrOracleConnectorPlugin.cs | 21 + ...ps.Feedser.Source.Vndr.Vmware.Tests.csproj | 13 + .../Vmware/VmwareMapperTests.cs | 83 + .../AGENTS.md | 29 + .../Configuration/VmwareOptions.cs | 54 + .../Internal/VmwareCursor.cs | 116 ++ .../Internal/VmwareDetailDto.cs | 53 + .../Internal/VmwareIndexItem.cs | 16 + .../Internal/VmwareMapper.cs | 150 ++ .../Jobs.cs | 46 + ...tellaOps.Feedser.Source.Vndr.Vmware.csproj | 23 + .../TASKS.md | 16 + .../VmwareConnector.cs | 374 ++++ .../VmwareConnectorPlugin.cs | 20 + .../VmwareDependencyInjectionRoutine.cs | 53 + .../VmwareServiceCollectionExtensions.cs | 37 + .../AdvisoryStorePerformanceTests.cs | 187 ++ .../AdvisoryStoreTests.cs | 45 + .../DocumentStoreTests.cs | 51 + .../DtoStoreTests.cs | 40 + .../ExportStateManagerTests.cs | 110 ++ .../ExportStateStoreTests.cs | 38 + .../MergeEventStoreTests.cs | 34 + .../Migrations/MongoMigrationRunnerTests.cs | 238 +++ .../MongoJobStoreTests.cs | 100 ++ .../MongoSourceStateRepositoryTests.cs | 55 + .../RawDocumentRetentionServiceTests.cs | 92 + ...ellaOps.Feedser.Storage.Mongo.Tests.csproj | 12 + src/StellaOps.Feedser.Storage.Mongo/AGENTS.md | 29 + .../Advisories/AdvisoryDocument.cs | 27 + .../Advisories/AdvisoryStore.cs | 245 +++ .../Advisories/IAdvisoryStore.cs | 14 + .../ChangeHistory/ChangeHistoryDocument.cs | 43 + .../ChangeHistoryDocumentExtensions.cs | 70 + .../ChangeHistory/ChangeHistoryFieldChange.cs | 24 + .../ChangeHistory/ChangeHistoryRecord.cs | 62 + .../ChangeHistory/IChangeHistoryStore.cs | 12 + .../ChangeHistory/MongoChangeHistoryStore.cs | 53 + .../Documents/DocumentDocument.cs | 130 ++ .../Documents/DocumentRecord.cs | 22 + .../Documents/DocumentStore.cs | 66 + .../Documents/IDocumentStore.cs | 12 + .../Dtos/DtoDocument.cs | 49 + .../Dtos/DtoRecord.cs | 11 + .../Dtos/DtoStore.cs | 55 + .../Dtos/IDtoStore.cs | 10 + .../Exporting/ExportStateDocument.cs | 63 + .../Exporting/ExportStateManager.cs | 102 ++ .../Exporting/ExportStateRecord.cs | 12 + .../Exporting/ExportStateStore.cs | 43 + .../Exporting/IExportStateStore.cs | 8 + .../ISourceStateRepository.cs | 14 + .../JobLeaseDocument.cs | 38 + .../JobRunDocument.cs | 119 ++ .../JpFlags/IJpFlagStore.cs | 11 + .../JpFlags/JpFlagDocument.cs | 54 + .../JpFlags/JpFlagRecord.cs | 15 + .../JpFlags/JpFlagStore.cs | 39 + .../MIGRATIONS.md | 37 + .../MergeEvents/IMergeEventStore.cs | 8 + .../MergeEvents/MergeEventDocument.cs | 48 + .../MergeEvents/MergeEventRecord.cs | 9 + .../MergeEvents/MergeEventStore.cs | 36 + .../EnsureDocumentExpiryIndexesMigration.cs | 146 ++ .../EnsureGridFsExpiryIndexesMigration.cs | 95 + .../Migrations/IMongoMigration.cs | 24 + .../Migrations/MongoMigrationDocument.cs | 18 + .../Migrations/MongoMigrationRunner.cs | 102 ++ .../MongoBootstrapper.cs | 306 ++++ .../MongoJobStore.cs | 192 ++ .../MongoLeaseStore.cs | 116 ++ .../MongoSourceStateRepository.cs | 112 ++ .../MongoStorageDefaults.cs | 28 + .../MongoStorageOptions.cs | 78 + .../Properties/AssemblyInfo.cs | 3 + .../PsirtFlags/IPsirtFlagStore.cs | 11 + .../PsirtFlags/PsirtFlagDocument.cs | 52 + .../PsirtFlags/PsirtFlagRecord.cs | 15 + .../PsirtFlags/PsirtFlagStore.cs | 50 + .../RawDocumentRetentionService.cs | 155 ++ .../ServiceCollectionExtensions.cs | 88 + .../SourceStateDocument.cs | 73 + .../SourceStateRecord.cs | 15 + .../SourceStateRepositoryExtensions.cs | 19 + .../StellaOps.Feedser.Storage.Mongo.csproj | 19 + src/StellaOps.Feedser.Storage.Mongo/TASKS.md | 15 + .../ConnectorTestHarness.cs | 118 ++ .../MongoIntegrationFixture.cs | 31 + .../StellaOps.Feedser.Testing.csproj | 18 + .../AssemblyInfo.cs | 3 + .../MongoFixtureCollection.cs | 6 + .../PluginLoaderTests.cs | 29 + .../StellaOps.Feedser.WebService.Tests.csproj | 13 + .../WebServiceEndpointsTests.cs | 346 ++++ src/StellaOps.Feedser.WebService/AGENTS.md | 36 + .../Diagnostics/HealthContracts.cs | 32 + .../Diagnostics/JobMetrics.cs | 25 + .../Diagnostics/ProblemTypes.cs | 12 + .../Diagnostics/ServiceStatus.cs | 74 + .../Extensions/ConfigurationExtensions.cs | 38 + .../Extensions/JobRegistrationExtensions.cs | 96 + .../Extensions/TelemetryExtensions.cs | 217 +++ .../Jobs/JobDefinitionResponse.cs | 23 + .../Jobs/JobRunResponse.cs | 29 + .../Jobs/JobTriggerRequest.cs | 8 + .../Options/FeedserOptions.cs | 53 + .../Options/FeedserOptionsValidator.cs | 54 + src/StellaOps.Feedser.WebService/Program.cs | 472 +++++ .../Properties/launchSettings.json | 12 + .../StellaOps.Feedser.WebService.csproj | 31 + src/StellaOps.Feedser.WebService/TASKS.md | 16 + src/StellaOps.Feedser.sln | 860 +++++++++ .../PluginDependencyInjectionExtensions.cs | 91 + .../StellaOpsPluginRegistration.cs | 26 + .../Hosting/PluginAssembly.cs | 21 + src/StellaOps.Plugin/Hosting/PluginHost.cs | 216 +++ .../Hosting/PluginHostOptions.cs | 59 + .../Hosting/PluginHostResult.cs | 26 + .../Hosting/PluginLoadContext.cs | 79 + .../Internal/ReflectionExtensions.cs | 21 + src/StellaOps.Plugin/PluginContracts.cs | 172 ++ src/StellaOps.Plugin/StellaOps.Plugin.csproj | 19 + src/farewell.txt | 1 + 621 files changed, 54480 insertions(+) create mode 100644 .gitea/workflows/_deprecated-feedser-ci.yml.disabled create mode 100644 .gitea/workflows/_deprecated-feedser-tests.yml.disabled create mode 100644 .gitea/workflows/build-test-deploy.yml create mode 100755 .gitea/workflows/docs.yml create mode 100644 .gitea/workflows/promote.yml create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100755 LICENSE create mode 100755 README.md create mode 100755 docs/01_WHAT_IS_IT.md create mode 100755 docs/02_WHY.md create mode 100755 docs/03_QUICKSTART.md create mode 100755 docs/03_VISION.md create mode 100755 docs/04_FEATURE_MATRIX.md create mode 100755 docs/05_ROADMAP.md create mode 100755 docs/05_SYSTEM_REQUIREMENTS_SPEC.md create mode 100755 docs/07_HIGH_LEVEL_ARCHITECTURE.md create mode 100755 docs/08_MODULE_SPECIFICATIONS.md create mode 100755 docs/09_API_CLI_REFERENCE.md create mode 100755 docs/10_OFFLINE_KIT.md create mode 100755 docs/10_PLUGIN_SDK_GUIDE.md create mode 100755 docs/11_DATA_SCHEMAS.md create mode 100755 docs/11_GOVERNANCE.md create mode 100755 docs/12_CODE_OF_CONDUCT.md create mode 100755 docs/12_PERFORMANCE_WORKBOOK.md create mode 100755 docs/13_RELEASE_ENGINEERING_PLAYBOOK.md create mode 100755 docs/13_SECURITY_POLICY.md create mode 100755 docs/14_GLOSSARY_OF_TERMS.md create mode 100755 docs/15_UI_GUIDE.md create mode 100755 docs/17_SECURITY_HARDENING_GUIDE.md create mode 100755 docs/18_CODING_STANDARDS.md create mode 100755 docs/19_TEST_SUITE_OVERVIEW.md create mode 100755 docs/21_INSTALL_GUIDE.md create mode 100755 docs/23_FAQ_MATRIX.md create mode 100755 docs/24_OFFLINE_KIT.md create mode 100755 docs/29_LEGAL_FAQ_QUOTA.md create mode 100755 docs/30_QUOTA_ENFORCEMENT_FLOW1.md create mode 100755 docs/33_333_QUOTA_OVERVIEW.md create mode 100755 docs/40_ARCHITECTURE_OVERVIEW.md create mode 100755 docs/60_POLICY_TEMPLATES.md create mode 100644 docs/ARCHITECTURE_FEEDSER.md create mode 100755 docs/README.md create mode 100755 docs/_includes/CONSTANTS.md create mode 100755 docs/ci/20_CI_RECIPES.md create mode 100755 docs/cli/20_REFERENCE.md create mode 100755 docs/dev/30_PLUGIN_DEV_GUIDE.md create mode 100755 docs/license-jwt-quota.md create mode 100644 global.json create mode 100644 scripts/render_docs.py create mode 100644 src/Directory.Build.props create mode 100644 src/Directory.Build.targets create mode 100644 src/Jobs.cs create mode 100644 src/OracleConnector.cs create mode 100644 src/OracleConnectorPlugin.cs create mode 100644 src/OracleDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.DependencyInjection/IDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj create mode 100644 src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs create mode 100644 src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj create mode 100644 src/StellaOps.Feedser.Core/AGENTS.md create mode 100644 src/StellaOps.Feedser.Core/Jobs/IJob.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/IJobStore.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobLease.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs create mode 100644 src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj create mode 100644 src/StellaOps.Feedser.Core/TASKS.md create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj create mode 100644 src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/AGENTS.md create mode 100644 src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs create mode 100644 src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj create mode 100644 src/StellaOps.Feedser.Exporter.Json/TASKS.md create mode 100644 src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs create mode 100644 src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs create mode 100644 src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj create mode 100644 src/StellaOps.Feedser.Merge.Tests/TestLogger.cs create mode 100644 src/StellaOps.Feedser.Merge/AGENTS.md create mode 100644 src/StellaOps.Feedser.Merge/Class1.cs create mode 100644 src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs create mode 100644 src/StellaOps.Feedser.Merge/Comparers/Nevra.cs create mode 100644 src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs create mode 100644 src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs create mode 100644 src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs create mode 100644 src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs create mode 100644 src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj create mode 100644 src/StellaOps.Feedser.Merge/TASKS.md create mode 100644 src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs create mode 100644 src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj create mode 100644 src/StellaOps.Feedser.Models/AGENTS.md create mode 100644 src/StellaOps.Feedser.Models/Advisory.cs create mode 100644 src/StellaOps.Feedser.Models/AdvisoryProvenance.cs create mode 100644 src/StellaOps.Feedser.Models/AdvisoryReference.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedPackage.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedPackageStatus.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs create mode 100644 src/StellaOps.Feedser.Models/AffectedVersionRange.cs create mode 100644 src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs create mode 100644 src/StellaOps.Feedser.Models/AliasSchemes.cs create mode 100644 src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md create mode 100644 src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md create mode 100644 src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs create mode 100644 src/StellaOps.Feedser.Models/CvssMetric.cs create mode 100644 src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md create mode 100644 src/StellaOps.Feedser.Models/SeverityNormalization.cs create mode 100644 src/StellaOps.Feedser.Models/SnapshotSerializer.cs create mode 100644 src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj create mode 100644 src/StellaOps.Feedser.Models/TASKS.md create mode 100644 src/StellaOps.Feedser.Models/Validation.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj create mode 100644 src/StellaOps.Feedser.Normalization/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs create mode 100644 src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs create mode 100644 src/StellaOps.Feedser.Normalization/Distro/Nevra.cs create mode 100644 src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs create mode 100644 src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs create mode 100644 src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs create mode 100644 src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj create mode 100644 src/StellaOps.Feedser.Normalization/TASKS.md create mode 100644 src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs create mode 100644 src/StellaOps.Feedser.Source.Acsc/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj create mode 100644 src/StellaOps.Feedser.Source.Cccs/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj create mode 100644 src/StellaOps.Feedser.Source.CertBund/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj create mode 100644 src/StellaOps.Feedser.Source.CertCc/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml create mode 100644 src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.CertFr/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj create mode 100644 src/StellaOps.Feedser.Source.CertFr/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json create mode 100644 src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.CertIn/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj create mode 100644 src/StellaOps.Feedser.Source.CertIn/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Common/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs create mode 100644 src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj create mode 100644 src/StellaOps.Feedser.Source.Common/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs create mode 100644 src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj create mode 100644 src/StellaOps.Feedser.Source.Ghsa/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj create mode 100644 src/StellaOps.Feedser.Source.Ics.Cisa/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj create mode 100644 src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Jvn/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd create mode 100644 src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj create mode 100644 src/StellaOps.Feedser.Source.Jvn/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Kev/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj create mode 100644 src/StellaOps.Feedser.Source.Kisa/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Nvd/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json create mode 100644 src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj create mode 100644 src/StellaOps.Feedser.Source.Nvd/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Osv/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/OsvConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj create mode 100644 src/StellaOps.Feedser.Source.Osv/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj create mode 100644 src/StellaOps.Feedser.Source.Ru.Nkcki/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Cisco/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Msrc/Class1.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs create mode 100644 src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj create mode 100644 src/StellaOps.Feedser.Storage.Mongo/AGENTS.md create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs create mode 100644 src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj create mode 100644 src/StellaOps.Feedser.Storage.Mongo/TASKS.md create mode 100644 src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs create mode 100644 src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs create mode 100644 src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj create mode 100644 src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs create mode 100644 src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs create mode 100644 src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs create mode 100644 src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj create mode 100644 src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs create mode 100644 src/StellaOps.Feedser.WebService/AGENTS.md create mode 100644 src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs create mode 100644 src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs create mode 100644 src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs create mode 100644 src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs create mode 100644 src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs create mode 100644 src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs create mode 100644 src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs create mode 100644 src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs create mode 100644 src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs create mode 100644 src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs create mode 100644 src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs create mode 100644 src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs create mode 100644 src/StellaOps.Feedser.WebService/Program.cs create mode 100644 src/StellaOps.Feedser.WebService/Properties/launchSettings.json create mode 100644 src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj create mode 100644 src/StellaOps.Feedser.WebService/TASKS.md create mode 100644 src/StellaOps.Feedser.sln create mode 100644 src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs create mode 100644 src/StellaOps.Plugin/DependencyInjection/StellaOpsPluginRegistration.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginAssembly.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginHost.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginHostOptions.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginHostResult.cs create mode 100644 src/StellaOps.Plugin/Hosting/PluginLoadContext.cs create mode 100644 src/StellaOps.Plugin/Internal/ReflectionExtensions.cs create mode 100644 src/StellaOps.Plugin/PluginContracts.cs create mode 100644 src/StellaOps.Plugin/StellaOps.Plugin.csproj create mode 100644 src/farewell.txt diff --git a/.gitea/workflows/_deprecated-feedser-ci.yml.disabled b/.gitea/workflows/_deprecated-feedser-ci.yml.disabled new file mode 100644 index 00000000..28abe340 --- /dev/null +++ b/.gitea/workflows/_deprecated-feedser-ci.yml.disabled @@ -0,0 +1,29 @@ +name: Feedser CI + +on: + push: + branches: ["main", "develop"] + pull_request: + branches: ["main", "develop"] + +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Setup .NET 10 preview + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.100-rc.1.25451.107 + include-prerelease: true + + - name: Restore dependencies + run: dotnet restore src/StellaOps.Feedser/StellaOps.Feedser.sln + + - name: Build + run: dotnet build src/StellaOps.Feedser/StellaOps.Feedser.sln --configuration Release --no-restore -warnaserror + + - name: Test + run: dotnet test src/StellaOps.Feedser/StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj --configuration Release --no-restore --logger "trx;LogFileName=feedser-tests.trx" diff --git a/.gitea/workflows/_deprecated-feedser-tests.yml.disabled b/.gitea/workflows/_deprecated-feedser-tests.yml.disabled new file mode 100644 index 00000000..8de9d7d1 --- /dev/null +++ b/.gitea/workflows/_deprecated-feedser-tests.yml.disabled @@ -0,0 +1,87 @@ +name: Feedser Tests CI + +on: + push: + paths: + - 'StellaOps.Feedser/**' + - '.gitea/workflows/feedser-tests.yml' + pull_request: + paths: + - 'StellaOps.Feedser/**' + - '.gitea/workflows/feedser-tests.yml' + +jobs: + advisory-store-performance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.100-rc.1 + + - name: Restore dependencies + working-directory: StellaOps.Feedser + run: dotnet restore StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj + + - name: Run advisory store performance test + working-directory: StellaOps.Feedser + run: | + set -euo pipefail + dotnet test \ + StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj \ + --filter "FullyQualifiedName~AdvisoryStorePerformanceTests" \ + --logger:"console;verbosity=detailed" | tee performance.log + + - name: Upload performance log + if: always() + uses: actions/upload-artifact@v4 + with: + name: advisory-store-performance-log + path: StellaOps.Feedser/performance.log + + full-test-suite: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.100-rc.1 + + - name: Restore dependencies + working-directory: StellaOps.Feedser + run: dotnet restore StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj + + - name: Run full test suite with baseline guard + working-directory: StellaOps.Feedser + env: + BASELINE_SECONDS: "19.8" + TOLERANCE_PERCENT: "25" + run: | + set -euo pipefail + start=$(date +%s) + dotnet test StellaOps.Feedser.Tests/StellaOps.Feedser.Tests.csproj --no-build | tee full-tests.log + end=$(date +%s) + duration=$((end-start)) + echo "Full test duration: ${duration}s" + export DURATION_SECONDS="$duration" + python - <<'PY' +import os, sys +duration = float(os.environ["DURATION_SECONDS"]) +baseline = float(os.environ["BASELINE_SECONDS"]) +tolerance = float(os.environ["TOLERANCE_PERCENT"]) +threshold = baseline * (1 + tolerance / 100) +print(f"Baseline {baseline:.1f}s, threshold {threshold:.1f}s, observed {duration:.1f}s") +if duration > threshold: + sys.exit(f"Full test duration {duration:.1f}s exceeded threshold {threshold:.1f}s") +PY + + - name: Upload full test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: full-test-suite-log + path: StellaOps.Feedser/full-tests.log diff --git a/.gitea/workflows/build-test-deploy.yml b/.gitea/workflows/build-test-deploy.yml new file mode 100644 index 00000000..4beeae2e --- /dev/null +++ b/.gitea/workflows/build-test-deploy.yml @@ -0,0 +1,297 @@ +# .gitea/workflows/build-test-deploy.yml +# Unified CI/CD workflow for git.stella-ops.org (Feedser monorepo) + +name: Build Test Deploy + +on: + push: + branches: [ main ] + paths: + - 'src/**' + - 'docs/**' + - 'scripts/**' + - 'Directory.Build.props' + - 'Directory.Build.targets' + - 'global.json' + - '.gitea/workflows/**' + pull_request: + branches: [ main, develop ] + paths: + - 'src/**' + - 'docs/**' + - 'scripts/**' + - '.gitea/workflows/**' + workflow_dispatch: + inputs: + force_deploy: + description: 'Ignore branch checks and run the deploy stage' + required: false + default: 'false' + type: boolean + +env: + DOTNET_VERSION: '10.0.100-rc.1.25451.107' + BUILD_CONFIGURATION: Release + CI_CACHE_ROOT: /data/.cache/stella-ops/feedser + RUNNER_TOOL_CACHE: /toolcache + +jobs: + build-test: + runs-on: ubuntu-22.04 + environment: ${{ github.event_name == 'pull_request' && 'preview' || 'staging' }} + env: + PUBLISH_DIR: ${{ github.workspace }}/artifacts/publish/webservice + TEST_RESULTS_DIR: ${{ github.workspace }}/artifacts/test-results + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup .NET ${{ env.DOTNET_VERSION }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + include-prerelease: true + + - name: Restore dependencies + run: dotnet restore src/StellaOps.Feedser.sln + + - name: Build solution (warnings as errors) + run: dotnet build src/StellaOps.Feedser.sln --configuration $BUILD_CONFIGURATION --no-restore -warnaserror + + - name: Run unit and integration tests + run: | + mkdir -p "$TEST_RESULTS_DIR" + dotnet test src/StellaOps.Feedser.sln \ + --configuration $BUILD_CONFIGURATION \ + --no-build \ + --logger "trx;LogFileName=stellaops-feedser-tests.trx" \ + --results-directory "$TEST_RESULTS_DIR" + + - name: Publish Feedser web service + run: | + mkdir -p "$PUBLISH_DIR" + dotnet publish src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj \ + --configuration $BUILD_CONFIGURATION \ + --no-build \ + --output "$PUBLISH_DIR" + + - name: Upload published artifacts + uses: actions/upload-artifact@v4 + with: + name: feedser-publish + path: ${{ env.PUBLISH_DIR }} + if-no-files-found: error + retention-days: 7 + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: feedser-test-results + path: ${{ env.TEST_RESULTS_DIR }} + if-no-files-found: ignore + retention-days: 7 + + docs: + runs-on: ubuntu-22.04 + env: + DOCS_OUTPUT_DIR: ${{ github.workspace }}/artifacts/docs-site + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install documentation dependencies + run: | + python -m pip install --upgrade pip + python -m pip install markdown pygments + + - name: Render documentation bundle + run: | + python scripts/render_docs.py --source docs --output "$DOCS_OUTPUT_DIR" --clean + + - name: Upload documentation artifact + uses: actions/upload-artifact@v4 + with: + name: feedser-docs-site + path: ${{ env.DOCS_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 7 + + deploy: + runs-on: ubuntu-22.04 + needs: [build-test, docs] + if: >- + needs.build-test.result == 'success' && + needs.docs.result == 'success' && + ( + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + github.event_name == 'workflow_dispatch' + ) + environment: staging + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: | + scripts + .gitea/workflows + sparse-checkout-cone-mode: true + + - name: Check if deployment should proceed + id: check-deploy + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ "${{ github.event.inputs.force_deploy }}" = "true" ]; then + echo "should-deploy=true" >> $GITHUB_OUTPUT + echo "✅ Manual deployment requested" + else + echo "should-deploy=false" >> $GITHUB_OUTPUT + echo "ℹ️ Manual dispatch without force_deploy=true — skipping" + fi + elif [ "${{ github.ref }}" = "refs/heads/main" ]; then + echo "should-deploy=true" >> $GITHUB_OUTPUT + echo "✅ Deploying latest main branch build" + else + echo "should-deploy=false" >> $GITHUB_OUTPUT + echo "ℹ️ Deployment restricted to main branch" + fi + + - name: Resolve deployment credentials + id: params + if: steps.check-deploy.outputs.should-deploy == 'true' + run: | + missing=() + + host="${{ secrets.STAGING_DEPLOYMENT_HOST }}" + if [ -z "$host" ]; then host="${{ vars.STAGING_DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ secrets.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ vars.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then missing+=("STAGING_DEPLOYMENT_HOST"); fi + + user="${{ secrets.STAGING_DEPLOYMENT_USERNAME }}" + if [ -z "$user" ]; then user="${{ vars.STAGING_DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ secrets.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ vars.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then missing+=("STAGING_DEPLOYMENT_USERNAME"); fi + + path="${{ secrets.STAGING_DEPLOYMENT_PATH }}" + if [ -z "$path" ]; then path="${{ vars.STAGING_DEPLOYMENT_PATH }}"; fi + + docs_path="${{ secrets.STAGING_DOCS_PATH }}" + if [ -z "$docs_path" ]; then docs_path="${{ vars.STAGING_DOCS_PATH }}"; fi + + key="${{ secrets.STAGING_DEPLOYMENT_KEY }}" + if [ -z "$key" ]; then key="${{ secrets.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.STAGING_DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then missing+=("STAGING_DEPLOYMENT_KEY"); fi + + if [ ${#missing[@]} -gt 0 ]; then + echo "❌ Missing deployment configuration: ${missing[*]}" + exit 1 + fi + + key_file="$RUNNER_TEMP/staging_deploy_key" + printf '%s\n' "$key" > "$key_file" + chmod 600 "$key_file" + + echo "host=$host" >> $GITHUB_OUTPUT + echo "user=$user" >> $GITHUB_OUTPUT + echo "path=$path" >> $GITHUB_OUTPUT + echo "docs-path=$docs_path" >> $GITHUB_OUTPUT + echo "key-file=$key_file" >> $GITHUB_OUTPUT + + - name: Download service artifact + if: steps.check-deploy.outputs.should-deploy == 'true' && steps.params.outputs.path != '' + uses: actions/download-artifact@v4 + with: + name: feedser-publish + path: artifacts/service + + - name: Download documentation artifact + if: steps.check-deploy.outputs.should-deploy == 'true' && steps.params.outputs['docs-path'] != '' + uses: actions/download-artifact@v4 + with: + name: feedser-docs-site + path: artifacts/docs + + - name: Install rsync + if: steps.check-deploy.outputs.should-deploy == 'true' + run: | + if command -v rsync >/dev/null 2>&1; then + exit 0 + fi + CACHE_DIR="${CI_CACHE_ROOT:-/tmp}/apt" + mkdir -p "$CACHE_DIR" + KEY="rsync-$(lsb_release -rs 2>/dev/null || echo unknown)" + DEB_DIR="$CACHE_DIR/$KEY" + mkdir -p "$DEB_DIR" + if ls "$DEB_DIR"/rsync*.deb >/dev/null 2>&1; then + apt-get update + apt-get install -y --no-install-recommends "$DEB_DIR"/libpopt0*.deb "$DEB_DIR"/rsync*.deb + else + apt-get update + apt-get download rsync libpopt0 + mv rsync*.deb libpopt0*.deb "$DEB_DIR"/ + dpkg -i "$DEB_DIR"/libpopt0*.deb "$DEB_DIR"/rsync*.deb || apt-get install -f -y + fi + + - name: Deploy service bundle + if: steps.check-deploy.outputs.should-deploy == 'true' && steps.params.outputs.path != '' + env: + HOST: ${{ steps.params.outputs.host }} + USER: ${{ steps.params.outputs.user }} + TARGET: ${{ steps.params.outputs.path }} + KEY_FILE: ${{ steps.params.outputs['key-file'] }} + run: | + SERVICE_DIR="artifacts/service/feedser-publish" + if [ ! -d "$SERVICE_DIR" ]; then + echo "❌ Service artifact directory missing ($SERVICE_DIR)" + exit 1 + fi + echo "🚀 Deploying Feedser web service to $HOST:$TARGET" + rsync -az --delete \ + -e "ssh -i $KEY_FILE -o StrictHostKeyChecking=no" \ + "$SERVICE_DIR"/ \ + "$USER@$HOST:$TARGET/" + + - name: Deploy documentation bundle + if: steps.check-deploy.outputs.should-deploy == 'true' && steps.params.outputs['docs-path'] != '' + env: + HOST: ${{ steps.params.outputs.host }} + USER: ${{ steps.params.outputs.user }} + DOCS_TARGET: ${{ steps.params.outputs['docs-path'] }} + KEY_FILE: ${{ steps.params.outputs['key-file'] }} + run: | + DOCS_DIR="artifacts/docs/feedser-docs-site" + if [ ! -d "$DOCS_DIR" ]; then + echo "❌ Documentation artifact directory missing ($DOCS_DIR)" + exit 1 + fi + echo "📚 Deploying documentation bundle to $HOST:$DOCS_TARGET" + rsync -az --delete \ + -e "ssh -i $KEY_FILE -o StrictHostKeyChecking=no" \ + "$DOCS_DIR"/ \ + "$USER@$HOST:$DOCS_TARGET/" + + - name: Deployment summary + if: steps.check-deploy.outputs.should-deploy == 'true' + run: | + echo "✅ Deployment completed" + echo " Host: ${{ steps.params.outputs.host }}" + echo " Service path: ${{ steps.params.outputs.path || '(skipped)' }}" + echo " Docs path: ${{ steps.params.outputs['docs-path'] || '(skipped)' }}" + + - name: Deployment skipped summary + if: steps.check-deploy.outputs.should-deploy != 'true' + run: | + echo "ℹ️ Deployment stage skipped" + echo " Event: ${{ github.event_name }}" + echo " Ref: ${{ github.ref }}" diff --git a/.gitea/workflows/docs.yml b/.gitea/workflows/docs.yml new file mode 100755 index 00000000..35572726 --- /dev/null +++ b/.gitea/workflows/docs.yml @@ -0,0 +1,70 @@ +# .gitea/workflows/docs.yml +# Documentation quality checks and preview artefacts + +name: Docs CI + +on: + push: + paths: + - 'docs/**' + - 'scripts/render_docs.py' + - '.gitea/workflows/docs.yml' + pull_request: + paths: + - 'docs/**' + - 'scripts/render_docs.py' + - '.gitea/workflows/docs.yml' + workflow_dispatch: {} + +env: + NODE_VERSION: '20' + PYTHON_VERSION: '3.11' + +jobs: + lint-and-preview: + runs-on: ubuntu-22.04 + env: + DOCS_OUTPUT_DIR: ${{ github.workspace }}/artifacts/docs-preview + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install markdown linters + run: | + npm install markdown-link-check remark-cli remark-preset-lint-recommended + + - name: Link check + run: | + find docs -name '*.md' -print0 | \ + xargs -0 -n1 -I{} npx markdown-link-check --quiet '{}' + + - name: Remark lint + run: | + npx remark docs -qf + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install documentation dependencies + run: | + python -m pip install --upgrade pip + python -m pip install markdown pygments + + - name: Render documentation preview bundle + run: | + python scripts/render_docs.py --source docs --output "$DOCS_OUTPUT_DIR" --clean + + - name: Upload documentation preview + if: always() + uses: actions/upload-artifact@v4 + with: + name: feedser-docs-preview + path: ${{ env.DOCS_OUTPUT_DIR }} + retention-days: 7 diff --git a/.gitea/workflows/promote.yml b/.gitea/workflows/promote.yml new file mode 100644 index 00000000..141dd228 --- /dev/null +++ b/.gitea/workflows/promote.yml @@ -0,0 +1,206 @@ +# .gitea/workflows/promote.yml +# Manual promotion workflow to copy staged artefacts to production + +name: Promote Feedser (Manual) + +on: + workflow_dispatch: + inputs: + include_docs: + description: 'Also promote the generated documentation bundle' + required: false + default: 'true' + type: boolean + tag: + description: 'Optional build identifier to record in the summary' + required: false + default: 'latest' + type: string + +jobs: + promote: + runs-on: ubuntu-22.04 + environment: production + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Resolve staging credentials + id: staging + run: | + missing=() + + host="${{ secrets.STAGING_DEPLOYMENT_HOST }}" + if [ -z "$host" ]; then host="${{ vars.STAGING_DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ secrets.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ vars.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then missing+=("STAGING_DEPLOYMENT_HOST"); fi + + user="${{ secrets.STAGING_DEPLOYMENT_USERNAME }}" + if [ -z "$user" ]; then user="${{ vars.STAGING_DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ secrets.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ vars.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then missing+=("STAGING_DEPLOYMENT_USERNAME"); fi + + path="${{ secrets.STAGING_DEPLOYMENT_PATH }}" + if [ -z "$path" ]; then path="${{ vars.STAGING_DEPLOYMENT_PATH }}"; fi + if [ -z "$path" ]; then missing+=("STAGING_DEPLOYMENT_PATH") + fi + + docs_path="${{ secrets.STAGING_DOCS_PATH }}" + if [ -z "$docs_path" ]; then docs_path="${{ vars.STAGING_DOCS_PATH }}"; fi + + key="${{ secrets.STAGING_DEPLOYMENT_KEY }}" + if [ -z "$key" ]; then key="${{ secrets.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.STAGING_DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then missing+=("STAGING_DEPLOYMENT_KEY"); fi + + if [ ${#missing[@]} -gt 0 ]; then + echo "❌ Missing staging configuration: ${missing[*]}" + exit 1 + fi + + key_file="$RUNNER_TEMP/staging_key" + printf '%s\n' "$key" > "$key_file" + chmod 600 "$key_file" + + echo "host=$host" >> $GITHUB_OUTPUT + echo "user=$user" >> $GITHUB_OUTPUT + echo "path=$path" >> $GITHUB_OUTPUT + echo "docs-path=$docs_path" >> $GITHUB_OUTPUT + echo "key-file=$key_file" >> $GITHUB_OUTPUT + + - name: Resolve production credentials + id: production + run: | + missing=() + + host="${{ secrets.PRODUCTION_DEPLOYMENT_HOST }}" + if [ -z "$host" ]; then host="${{ vars.PRODUCTION_DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ secrets.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then host="${{ vars.DEPLOYMENT_HOST }}"; fi + if [ -z "$host" ]; then missing+=("PRODUCTION_DEPLOYMENT_HOST"); fi + + user="${{ secrets.PRODUCTION_DEPLOYMENT_USERNAME }}" + if [ -z "$user" ]; then user="${{ vars.PRODUCTION_DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ secrets.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then user="${{ vars.DEPLOYMENT_USERNAME }}"; fi + if [ -z "$user" ]; then missing+=("PRODUCTION_DEPLOYMENT_USERNAME"); fi + + path="${{ secrets.PRODUCTION_DEPLOYMENT_PATH }}" + if [ -z "$path" ]; then path="${{ vars.PRODUCTION_DEPLOYMENT_PATH }}"; fi + if [ -z "$path" ]; then missing+=("PRODUCTION_DEPLOYMENT_PATH") + fi + + docs_path="${{ secrets.PRODUCTION_DOCS_PATH }}" + if [ -z "$docs_path" ]; then docs_path="${{ vars.PRODUCTION_DOCS_PATH }}"; fi + + key="${{ secrets.PRODUCTION_DEPLOYMENT_KEY }}" + if [ -z "$key" ]; then key="${{ secrets.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.PRODUCTION_DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then key="${{ vars.DEPLOYMENT_KEY }}"; fi + if [ -z "$key" ]; then missing+=("PRODUCTION_DEPLOYMENT_KEY"); fi + + if [ ${#missing[@]} -gt 0 ]; then + echo "❌ Missing production configuration: ${missing[*]}" + exit 1 + fi + + key_file="$RUNNER_TEMP/production_key" + printf '%s\n' "$key" > "$key_file" + chmod 600 "$key_file" + + echo "host=$host" >> $GITHUB_OUTPUT + echo "user=$user" >> $GITHUB_OUTPUT + echo "path=$path" >> $GITHUB_OUTPUT + echo "docs-path=$docs_path" >> $GITHUB_OUTPUT + echo "key-file=$key_file" >> $GITHUB_OUTPUT + + - name: Install rsync + run: | + if command -v rsync >/dev/null 2>&1; then + exit 0 + fi + CACHE_DIR="${CI_CACHE_ROOT:-/tmp}/apt" + mkdir -p "$CACHE_DIR" + KEY="rsync-$(lsb_release -rs 2>/dev/null || echo unknown)" + DEB_DIR="$CACHE_DIR/$KEY" + mkdir -p "$DEB_DIR" + if ls "$DEB_DIR"/rsync*.deb >/dev/null 2>&1; then + apt-get update + apt-get install -y --no-install-recommends "$DEB_DIR"/libpopt0*.deb "$DEB_DIR"/rsync*.deb + else + apt-get update + apt-get download rsync libpopt0 + mv rsync*.deb libpopt0*.deb "$DEB_DIR"/ + dpkg -i "$DEB_DIR"/libpopt0*.deb "$DEB_DIR"/rsync*.deb || apt-get install -f -y + fi + + - name: Fetch staging artefacts + id: fetch + run: | + staging_root="${{ runner.temp }}/staging" + mkdir -p "$staging_root/service" "$staging_root/docs" + + echo "📥 Copying service bundle from staging" + rsync -az --delete \ + -e "ssh -i ${{ steps.staging.outputs['key-file'] }} -o StrictHostKeyChecking=no" \ + "${{ steps.staging.outputs.user }}@${{ steps.staging.outputs.host }}:${{ steps.staging.outputs.path }}/" \ + "$staging_root/service/" + + if [ "${{ github.event.inputs.include_docs }}" = "true" ] && [ -n "${{ steps.staging.outputs['docs-path'] }}" ]; then + echo "📥 Copying documentation bundle from staging" + rsync -az --delete \ + -e "ssh -i ${{ steps.staging.outputs['key-file'] }} -o StrictHostKeyChecking=no" \ + "${{ steps.staging.outputs.user }}@${{ steps.staging.outputs.host }}:${{ steps.staging.outputs['docs-path'] }}/" \ + "$staging_root/docs/" + else + echo "ℹ️ Documentation promotion skipped" + fi + + echo "service-dir=$staging_root/service" >> $GITHUB_OUTPUT + echo "docs-dir=$staging_root/docs" >> $GITHUB_OUTPUT + + - name: Backup production service content + run: | + ssh -o StrictHostKeyChecking=no -i "${{ steps.production.outputs['key-file'] }}" \ + "${{ steps.production.outputs.user }}@${{ steps.production.outputs.host }}" \ + "set -e; TARGET='${{ steps.production.outputs.path }}'; \ + if [ -d \"$TARGET\" ]; then \ + parent=\$(dirname \"$TARGET\"); \ + base=\$(basename \"$TARGET\"); \ + backup=\"\$parent/\${base}.backup.\$(date +%Y%m%d_%H%M%S)\"; \ + mkdir -p \"\$backup\"; \ + rsync -a --delete \"$TARGET/\" \"\$backup/\"; \ + ls -dt \"\$parent/\${base}.backup.*\" 2>/dev/null | tail -n +6 | xargs rm -rf || true; \ + echo 'Backup created at ' \"\$backup\"; \ + else \ + echo 'Production service path missing; skipping backup'; \ + fi" + + - name: Publish service to production + run: | + rsync -az --delete \ + -e "ssh -i ${{ steps.production.outputs['key-file'] }} -o StrictHostKeyChecking=no" \ + "${{ steps.fetch.outputs['service-dir'] }}/" \ + "${{ steps.production.outputs.user }}@${{ steps.production.outputs.host }}:${{ steps.production.outputs.path }}/" + + - name: Promote documentation bundle + if: github.event.inputs.include_docs == 'true' && steps.production.outputs['docs-path'] != '' + run: | + rsync -az --delete \ + -e "ssh -i ${{ steps.production.outputs['key-file'] }} -o StrictHostKeyChecking=no" \ + "${{ steps.fetch.outputs['docs-dir'] }}/" \ + "${{ steps.production.outputs.user }}@${{ steps.production.outputs.host }}:${{ steps.production.outputs['docs-path'] }}/" + + - name: Promotion summary + run: | + echo "✅ Promotion completed" + echo " Tag: ${{ github.event.inputs.tag }}" + echo " Service: ${{ steps.staging.outputs.host }} → ${{ steps.production.outputs.host }}" + if [ "${{ github.event.inputs.include_docs }}" = "true" ]; then + echo " Docs: included" + else + echo " Docs: skipped" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..49435111 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Build outputs +bin/ +obj/ +*.pdb +*.dll + +# IDE state +.vs/ +*.user +*.suo +*.userprefs + +# Rider/VSCode +.idea/ +.vscode/ + +# Packages and logs +*.log +TestResults/ + +.dotnet \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..8764b86d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,125 @@ +# 1) What is StellaOps? + +**StellaOps** an open, sovereign, modular container-security toolkit built for high-speed, offline operation, released under AGPL-3.0-or-later. + +It follows an SBOM-first model—analyzing each container layer or ingesting existing CycloneDX/SPDX SBOMs, then enriching them with vulnerability, licence, secret-leak, and misconfiguration data to produce cryptographically signed reports. + +Vulnerability detection maps OS and language dependencies to sources such as NVD, GHSA, OSV, ENISA. +Secrets sweep flags exposed credentials or keys in files or environment variables. +Licence audit identifies potential conflicts, especially copyleft obligations. +Misconfiguration checks detect unsafe Dockerfile patterns (root user, latest tags, permissive modes). +Provenance features include in-toto/SLSA attestations signed with cosign for supply-chain trust. + +| Guiding principle | What it means for Feedser | +|-------------------|---------------------------| +| **SBOM-first ingest** | Prefer signed SBOMs or reproducible layer diffs before falling back to raw scraping; connectors treat source docs as provenance, never as mutable truth. | +| **Deterministic outputs** | Same inputs yield identical canonical advisories and exported JSON/Trivy DB artefacts; merge hashes and export manifests are reproducible across machines. | +| **Restart-time plug-ins only** | Connector/exporter plug-ins load at service start, keeping runtime sandboxing simple and avoiding hot-patch attack surface. | +| **Sovereign/offline-first** | No mandatory outbound calls beyond allow-listed advisories; Offline Kit bundles Mongo snapshots and exporter artefacts for air-gapped installs. | +| **Operational transparency** | Every stage logs structured events (fetch, parse, merge, export) with correlation IDs so parallel agents can debug without shared state. | + +Performance: warm scans < 5 s, cold scans < 30 s on a 4 vCPU runner. +Deployment: entirely SaaS-free, suitable for air-gapped or on-prem use through its Offline Kit. +Policy: anonymous users → 33 scans/day; verified → 333 /day; nearing 90 % quota triggers throttling but never full blocks. + +More documention is available ./docs/*.md files. Read `docs/README.md` to gather information about the available documentation. You could inquiry specific documents as your work requires it + +--- + +# 3) Practices + +## 3.1) Naming +All modules are .NET projects based on .NET 10 (preview). Exclussion is the UI. It is based on Angular +All modules are contained by one or more projects. Each project goes in its dedicated folder. Each project starts with StellaOps.. In case it is common for for all StellaOps modules it is library or plugin and it is named StellaOps.. + +## 3.2) Key technologies & integrations + +- **Runtime**: .NET 10 (`net10.0`) preview SDK; C# latest preview features. +- **Data**: MongoDB (canonical store and job/export state). +- **Observability**: structured logs, counters, and (optional) OpenTelemetry traces. +- **Ops posture**: offline‑first, allowlist for remote hosts, strict schema validation, gated LLM fallback (only where explicitly configured). + +# 4) Modules +StellaOps is contained by different modules installable via docker containers +- Feedser. Responsible for aggregation and delivery of vulnerability database +- Cli. Command line tool to unlock full potential - request database operations, install scanner, request scan, configure backend +- Backend. Configures and Manages scans +- UI. UI to access the backend (and scanners) +- Agent. Installable daemon that does the scanning +- Zastava. Realtime monitor for allowed (verified) installations. + +## 4.1) Feedser +It is webservice based module that is responsible for aggregating vulnerabilities information from various sources, parsing and normalizing them into a canonical shape, merging and deduplicating the results in one place, with export capabilities to Json and TrivyDb. It supports init and resume for all of the sources, parse/normalize and merge/deduplication operations, plus export. Export supports delta exports—similarly to full and incremential database backups. + +### 4.1.1) Usage +It supports operations to be started by cmd line: +# stella db [fetch|merge|export] [init|resume ] +or +api available on https://db.stella-ops.org + +### 4.1.2) Data flow (end‑to‑end) + +1. **Fetch**: connectors request source windows with retries/backoff, persist raw documents with SHA256/ETag metadata. +2. **Parse & Normalize**: validate to DTOs (schema-checked), quarantine failures, normalize to canonical advisories (aliases, affected ranges with NEVRA/EVR/SemVer, references, provenance). +3. **Merge & Deduplicate**: enforce precedence, build/maintain alias graphs, compute deterministic hashes, and eliminate duplicates before persisting to MongoDB. +4. **Export**: JSON tree and/or Trivy DB; package and (optionally) push; write export state. + +### 4.1.3) Architecture +For more information of the architecture see `./docs/ARCHITECTURE_FEEDSER.md`. + +--- + +### 4.1.4) Glossary (quick) + +- **OVAL** — Vendor/distro security definition format; authoritative for OS packages. +- **NEVRA / EVR** — RPM and Debian version semantics for OS packages. +- **PURL / SemVer** — Coordinates and version semantics for OSS ecosystems. +- **KEV** — Known Exploited Vulnerabilities (flag only). + +--- +# 5) Your role as StellaOps contributor + +You acting as information technology engineer that will take different type of roles in goal achieving StellaOps production implementation +In order you to work - you have to be supplied with directory that contains `AGENTS.md`,`TASKS.md` files. There will you have more information about the role you have, the scope of your work and the tasks you will have. + +Boundaries: +- You operate only in the working directories I gave you, unless there is dependencies that makes you to work on dependency in shared directory. Then you ask for confirmation. + +You main characteristics: +- Keep endpoints small, deterministic, and cancellation-aware. +- Improve logs/metrics as per tasks. +- Update `TASKS.md` when moving tasks forward. +- When you are done with all task you state explicitly you are done. +- Impersonate the role described on working directory `AGENTS.md` you will read, if role is not available - take role of the CTO of the StellaOps in early stages. +- You always strive for best practices +- You always strive for re-usability +- When in doubt of design decision - you ask then act +- You are autonomus - meaning that you will work for long time alone and achieve maximum without stopping for stupid questions +- You operate on the same directory where other agents will work. In case you need to work on directory that is dependency on provided `AGENTS.md`,`TASKS.md` files you have to ask for confirmation first. + +## 5.1) Type of contributions + +- **BE‑Base (Platform & Pipeline)** + Owns DI, plugin host, job scheduler/coordinator, configuration binding, minimal API endpoints, and Mongo bootstrapping. +- **BE‑Conn‑X (Connectors)** + One agent per source family (NVD, Red Hat, Ubuntu, Debian, SUSE, GHSA, OSV, PSIRTs, CERTs, KEV, ICS). Implements fetch/parse/map with incremental watermarks. +- **BE‑Merge (Canonical Merge & Dedupe)** + Identity graph, precedence policies, canonical JSON serializer, and deterministic hashing (`merge_event`). +- **BE‑Export (JSON & Trivy DB)** + Deterministic export trees, Trivy DB packaging, optional ORAS push, and offline bundle. +- **QA (Validation & Observability)** + Schema tests, fixture goldens, determinism checks, metrics/logs/traces, e2e reproducibility runs. +- **DevEx/Docs** + Maintains this agent framework, templates, and per‑directory guides; assists parallelization and reviews. + + +## 5.2) Work-in-parallel rules (important) + +- **Directory ownership**: Each agent works **only inside its module directory**. Cross‑module edits require a brief handshake in issues/PR description. +- **Scoping**: Use each module’s `AGENTS.md` and `TASKS.md` to plan; autonomous agents must read `src/AGENTS.md` and the module docs before acting. +- **Determinism**: Sort keys, normalize timestamps to UTC ISO‑8601, avoid non‑deterministic data in exports and tests. +- **Status tracking**: Update your module’s `TASKS.md` as you progress (TODO → DOING → DONE/BLOCKED). +- **Tests**: Add/extend fixtures and unit tests per change; never regress determinism or precedence. +- **Test layout**: Use module-specific projects in `StellaOps.Feedser..Tests`; shared fixtures/harnesses live in `StellaOps.Feedser.Testing`. + +--- diff --git a/LICENSE b/LICENSE new file mode 100755 index 00000000..0cabe4cc --- /dev/null +++ b/LICENSE @@ -0,0 +1,235 @@ +GNU AFFERO GENERAL PUBLIC LICENSE +Version 3, 19 November 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + + Preamble + +The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. + +The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. + +When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. + +Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. + +A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. + +The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. + +An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. + +The precise terms and conditions for copying, distribution and modification follow. + + TERMS AND CONDITIONS + +0. Definitions. + +"This License" refers to version 3 of the GNU Affero General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based on the Program. + +To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. + +1. Source Code. +The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. + +A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +2. Basic Permissions. +All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. +No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. + +4. Conveying Verbatim Copies. +You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. +You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". + + c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. + +6. Conveying Non-Source Forms. +You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: + + a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. + +7. Additional Terms. +"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. + +All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. + +8. Termination. + +You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. + +9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. + +11. Patents. + +A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. + +A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. + +13. Remote Network Interaction; Use with the GNU General Public License. + +Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. + +Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. + +14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. + +15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. + + git.stella-ops.org + Copyright (C) 2025 stella-ops.org + + This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. + +You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . diff --git a/README.md b/README.md new file mode 100755 index 00000000..585a1023 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# git.stella-ops.org + diff --git a/docs/01_WHAT_IS_IT.md b/docs/01_WHAT_IS_IT.md new file mode 100755 index 00000000..90e2bf86 --- /dev/null +++ b/docs/01_WHAT_IS_IT.md @@ -0,0 +1,77 @@ +# 1 · What Is - **Stella Ops**? + +Stella Ops is a **self‑hosted, SBOM‑first DevSecOps platform** that gives engineering and security teams instant (< 5 s) feedback on container and artifact risk—even when they run completely offline. +It is built around five design pillars: **modular, open, fast, local, and UI‑controllable**. + +--- + +## 1. What the Product Does — 7‑Point Snapshot + +| # | Capability | What It Means in Practice | +|---|------------|---------------------------| +| **1** | **SBOM‑Centric Scanning** | Generates and scans *Software Bills of Materials* (Trivy JSON, SPDX‑JSON, CycloneDX‑JSON); auto‑detects format and stores each SBOM as a blob. | +| **2** | **Delta‑SBOM Engine** | Uploads SBOM only for *new* layers; warm‑cache image rescans complete in < 1 s. | +| **3** | **Anonymous Internal Registry** | Ships a built‑in `StellaOps.Registry` so agents (`Stella CLI`, `Zastava`, SBOM‑builder) can be pulled inside air‑gapped networks without external credentials. | +| **4** | **Policy‑as‑Code** | Supports YAML rules today and OPA/Rego (`StellaOps.MutePolicies`) tomorrow—edit in the web UI, versioned in Mongo, enforce at scan time. | +| **5** | **Pluggable Modules** | Every scanner, exporter, or attestor is a hot‑load .NET plug‑in (e.g., `StellaOpsAttestor` for SLSA/Rekor in the roadmap). | +| **6** | **Horizontally Scalable** | Stateless API backed by Redis & Mongo; optional Kubernetes charts for multi‑node performance. | +| **7** | **Sovereign & Localized** | Localized UI, optional connectors to regional catalogues, and zero telemetry by default—ready for high‑compliance, air‑gapped deployments. | + +> **🆓 Free tier update (July 2025)** – Every self‑hosted instance now includes **{{ quota_token }} scans per UTC day**. +> A yellow banner appears once you cross **200 scans** (≈ 60 % of quota). +> Past {{ quota_token }} , `/scan` responds with soft 5 s waits (graceful back‑off), and may return **429 + Retry‑After (to UTC midnight)** after repeated hits. + +--- + +## 2. How It Works — End‑to‑End Flow (30 sec tour) + +1. **Build Phase** + `sbom‑builder` container runs inside CI, pulls base layers metadata, and queries `/layers/missing`—receiving in ~20 ms which layers still need SBOMs. + • New layers ➟ SBOM generated ➟ `*.sbom.` + `*.sbom.type` dropped next to image tarball. + +2. **Push to Registry** + Image and SBOM blobs are pushed to the **anonymous internal registry** (`StellaOps.Registry`). Cosign tags are attached if enabled. + +3. **Scan Phase** + `Stella CLI` agent pulls the SBOM blob, sends `/scan?sbomType=spdx-json` to backend. If flag is absent, backend auto‑detects. + • Free‑tier tokens inherit the **333‑scan/day quota**; response headers expose remaining scans and reset time. + +4. **Policy & Risk Evaluation** + Backend hydrates CVE data, merges any cached layer scores, and calls the **Policy‑as‑Code engine**: + * YAML rules → built‑in interpreter; + * Rego policies (future) → embedded OPA. + +5. **Attestation & Transparency** *(Roadmap)* + `StellaOpsAttestor` signs results with SLSA provenance and records them in a local **Rekor** mirror for tamper‑proof history. + +6. **Feedback Loop** + • CLI exits with non‑zero on policy block. + • UI dashboard shows findings, quota banner, and per‑token scan counters; triagers can mute or set expiry dates directly. + +--- + +## 3. Why Such a Product Is Needed + +> *“Software supply‑chain attacks have increased **742 %** over the past three years.”* – Sonatype 2024 State of the Software Supply Chain + +### Key Drivers & Regulations + +| Driver | Detail & Obligation | +|--------|--------------------| +| **Government SBOM Mandates** | • **US EO 14028** & NIST SP 800‑218 require suppliers to provide SBOMs.
• EU **Cyber Resilience Act (CRA)** will demand attestations of secure development by 2026. | +| **SLSA & SSDF Frameworks** | Industry pushes toward **SLSA v1.0** levels 2‑3 and NIST **SSDF 1.1** controls, emphasising provenance and policy enforcement. | +| **Transparency Logs** | **Sigstore Rekor** gains traction as a standard for tamper‑evident signatures—even for air‑gapped replicas. | +| **Offline & Sovereign Deployments** | Critical‑infra operators (finance, telecom, defence) must run security tooling without Internet and with local language/VDB support. | +| **Performance Expectations** | Modern CI/CD pipelines trigger hundreds of image builds daily; waiting 30‑60 s per scan is no longer acceptable—and now **must be achieved within a 333‑scan/day free quota**. | + +### Gap in Existing Tools + +* SaaS‑only scanners can’t run in regulated or disconnected environments. +* Monolithic open‑source scanners are hard‑wired to Trivy or Syft formats, lacking delta optimisation. +* Few products expose **Policy‑as‑Code** with full UI editing **and** history audit in a single package. +* None address quota‑aware throttling without hidden paywalls. + +**Stella Ops** fills this gap by combining *speed*, *modular openness*, *sovereign readiness* **and transparent quota limits**—making thorough supply‑chain security attainable for every team, not just cloud‑native startups. + +--- +*Last updated: 14 Jul 2025* diff --git a/docs/02_WHY.md b/docs/02_WHY.md new file mode 100755 index 00000000..21b83e94 --- /dev/null +++ b/docs/02_WHY.md @@ -0,0 +1,121 @@ +# 2 · WHY — Why Stella Ops Exists + +> Explaining the concrete pain we solve, why the world needs **one more** DevSecOps +> platform, and the success signals that prove we are on the right track. + +Software‑supply‑chain attacks, licence‑risk, and incomplete SBOM coverage slow +teams and compliance audits to a crawl. Most existing scanners: + +* **Assume Internet** access for CVE feeds or SaaS back‑ends. +* **Parse an entire image** every build (no layer‑delta optimisation). +* **Accept a single SBOM format** (usually Trivy JSON) and choke on anything else. +* Offer **no built‑in policy history / audit trail**. +* Require 30‑60 s wall‑time per scan, an order of magnitude slower than modern CI + expectations. +* **Hide quota limits** or throttle without warning once you move past free trials. + +--- +# 1 Free‑Tier Quota — Why **{{ quota_token }} **? + +The limit of **{{ quota_token }} SBOM scans per UTC day** was not chosen at random. + +| Constraint | Analysis | Outcome | +|------------|----------|---------| +| **SMB workload** | Internal survey across 37 SMBs shows median **210** container builds/day (p95 ≈ 290). | {{ quota_token }} gives ≈ 1.6 × head‑room without forcing a paid tier. | +| **Cost of feeds** | Hosting, Trivy DB mirrors & CVE merge traffic average **≈ $14 / 1 000 scans**. | {{ quota_token }} /day yields <$5 infra cost per user — sustainable for an OSS project. | +| **Incentive to upgrade** | Larger orgs (> 300 builds/day) gain ROI from Plus/Pro tiers anyway. | Clear upsell path without hurting hobbyists. | + +> **In one sentence:**  *{{ quota_token }} scans cover the daily needs of a typical small / +> medium business, keep free usage genuinely useful and still leave a financial +> runway for future development*. + +## 1.1 How the Quota Is Enforced (1‑minute view) + +* Backend loads the **Quota plug‑in** at startup. +* Every `/scan` call passes the caller’s **Client‑JWT** to the plug‑in. +* The plug‑in **increments a counter in Redis** under + `quota::` (expires at UTC midnight). +* Soft wait‑wall (5 s) after limit; hard wait‑wall (60 s) after 30 blocked calls. +* For **offline installs**, a *1‑month validity Client‑JWT* ships inside every + **Offline Update Kit (OUK)** tarball. Uploading the OUK refreshes the token + automatically. + +Detailed sequence living in **30_QUOTA_ENFORCEMENT_FLOW.md**. + + + +--- + +## 2 · Why *Another* DevSecOps Product? — Macro Drivers + +| Driver | Evidence | Implication for Tooling | +|--------|----------|-------------------------| +| **Exploding supply‑chain attacks** | Sonatype 2024 report shows **742 %** growth since 2020. | SBOMs & provenance checks must be default, not “best‑practice”. | +| **Regulation tsunami** | • US EO 14028 & NIST SP 800‑218
• EU Cyber‑Resilience Act (CRA) in force 2026
• Local critical‑infrastructure rules in some jurisdictions | Vendors must *attest* build provenance (SLSA) and store tamper‑proof SBOMs. | +| **Runtime‑cost intolerance** | Pipelines build hundreds of images/day; waiting > 10 s per scan breaks SLA. | Need **delta‑aware** engines that reuse layer analyses (< 1 s warm scans). | +| **Air‑gap & sovereignty demands** | Finance/defence prohibit outbound traffic; data must stay on‑prem. | Ship **self‑contained registry + CVE DB** and run offline. | +| **Predictable free‑tier limits** | Teams want clarity, not surprise throttling. | Provide **transparent {{ quota_token }} scans/day quota**, early banner & graceful wait‑wall. | + +> **Therefore:** The market demands a **modular, SBOM‑first, sub‑5 s, 100 % self‑hosted** +> platform **with a transparent free‑tier quota**—precisely the niche Stella Ops targets. + +--- + +## 3 · Gap in Current Tooling + +* Trivy / Syft create SBOMs but re‑analyse **every** layer → wasted minutes/day. +* Policy engines (OPA/Rego) are separate binaries, with no UI or change history. +* No mainstream OSS bundle ships an **anonymous internal registry** for air‑gapped pulls. +* Provenance attestation (SLSA) and Rekor transparency logs remain “bring‑your‑own”. +* Free tiers either stop at 100 scans **or** silently throttle; none announce a **clear {{ quota_token }} /day allowance**. + +--- + +## 4 · Why Stella Ops Can Win + +1. **Speed First** — Delta‑SBOM flow uses cached layers to hit `< 1 s` warm scans. +2. **Multi‑Format Ready** — Auto‑detects Trivy‑JSON, SPDX‑JSON, CycloneDX‑JSON; UI + lets teams choose per‑project defaults. +3. **Offline by Default** — Ships an **anonymous internal Docker registry** + (`StellaOps.Registry`) plus Redis, Mongo, CVE DB, and UI in a single compose up. +4. **Open & Modular** — .NET hot‑load plug‑ins (`StellaOpsAttestor`, future scanners) + under AGPL; anyone can extend. +5. **Policy as Code** — YAML rules today, upgrade path to OPA/Rego with history stored + in Mongo via `StellaOps.MutePolicies`. +6. **Sovereign‑Ready** — Russian‑language UI, local vulnerability mirrors, zero + telemetry by default. +7. **Honest Free‑tier Boundaries** — Clear **{{ quota_token }} scans/day** limit, early banner at 200 and predictable wait‑wall—no hidden throttling. + +--- + +## 5 · Success Criteria — Signals We Solve the Problem + +* **Performance:** P95 scan < 5 s on first pass; `< 1 s` for warm delta scans. +* **Compatibility:** SBOMs in at least three formats consumed by ≥ 3 downstream tools. +* **Adoption:** ≥ 1 000 reported installs & ≥ 2 000 binary downloads by Q2‑2026. +* **Compliance:** Positive audits referencing CRA / NIST / SLSA readiness. +* **Community:** ≥ 15 first‑time contributors merged per quarter by 2026. +* **Transparency:** 0 support tickets complaining about “mystery throttling”. + +--- + +## 6 · Non‑Goals (2025‑2027) + +* Multi‑tenant SaaS offering. +* Automatic “fix‑PR” generation (left to ecosystem). +* Windows container **scanning** (Windows *agents* are on the 12‑month roadmap). + +--- + +## 7 · Stakeholder Pain‑Point Recap + +| Persona | Pain Today | Stella Ops Solution | +|---------|------------|---------------------| +| **Dev** | “My CI fails for 45 s on every push.” | < 5 s initial, < 1 s warm scans. | +| **Sec‑Ops** | Separate tools for SBOM, policy, and audit. | Unified UI + YAML / Rego policies with history. | +| **Infra** | Internet‑blocked site; no public pulls allowed. | Offline compose bundle + internal registry. | +| **Compliance** | Need CRA‑ready provenance by 2026. | Future `StellaOpsAttestor` SLSA + Rekor integration. | +| **Budget owner** | Fears hidden overage charges in “free” tiers. | Transparent {{ quota_token }} scans/day limit, visible in UI/API. | + +--- +*Last updated: 14 Jul 2025 (sync with free‑tier quota rev 2.0).* diff --git a/docs/03_QUICKSTART.md b/docs/03_QUICKSTART.md new file mode 100755 index 00000000..900954cb --- /dev/null +++ b/docs/03_QUICKSTART.md @@ -0,0 +1,156 @@ +# Five‑Minute Quick‑Start ⚡ +Run your first container scan locally + +> **Heads‑up** – the public α `v0.1.0` image drops **late 2025**. +> Once it is published as +> `registry.stella-ops.org/stella-ops/stella-ops:0.1.0‑alpha` +> every command on this page works without changes. + +--- + +## 0 · What you need 🔧 + +| Requirement | Minimum | Notes | +|-------------|---------|-------| +| OS | Ubuntu 22.04 • Alma 9 | x86‑64 or arm64 | +| Docker | Engine 25 • Compose v2 | `docker -v` | +| CPU / RAM | 2 vCPU / 2 GiB | Dev‑laptop baseline | +| Disk | 10 GiB SSD | SBOM cache | + +> **Tip –** If you already have Redis & MongoDB, skip the infra +> compose file and point Stella Ops at those hosts via `.env`. + +--- + +## 1 · Fetch the signed Compose bundles 📦 + +```bash +# Infrastructure (Redis + MongoDB) +curl -LO https://get.stella-ops.org/docker-compose.infrastructure.yml +curl -LO https://get.stella-ops.org/docker-compose.infrastructure.yml.sig + +# Core scanner stack +curl -LO https://get.stella-ops.org/docker-compose.stella-ops.yml +curl -LO https://get.stella-ops.org/docker-compose.stella-ops.yml.sig + +# Verify signatures (supply‑chain 101) +cosign verify-blob --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.infrastructure.yml.sig docker-compose.infrastructure.yml +cosign verify-blob --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.stella-ops.yml.sig docker-compose.stella-ops.yml +```` + +--- + +## 2 · Create `.env` 🗝️ + +```bash + +# ─── Identity (shows in reports) ─────────────────────────── +STELLA_OPS_COMPANY_NAME="Acme Corp" +STELLA_OPS_ISSUER_EMAIL="ops@acme.example" +STELLA_OPS_DEFAULT_ADMIN_USERNAME="admin" +STELLA_OPS_DEFAULT_ADMIN_PASSWORD="changeme!" +STELLA_OPS_DEFAULT_JWT="" # or load it later with +# docker --env-file .env compose -f docker-compose.stella-ops.yml exec stella set-jwt + + +# ─── Database secrets ────────────────────────────────────── +MONGO_INITDB_ROOT_USERNAME=stella_admin +MONGO_INITDB_ROOT_PASSWORD=$(openssl rand -base64 18) +MONGO_URL=mongodb + +REDIS_PASSWORD=$(openssl rand -base64 18) +REDIS_URL=redis + + + +``` + +--- + +## 3 · Start the supporting services 🗄️ + +```bash +docker compose --env-file .env -f docker-compose.infrastructure.yml pull +docker compose --env-file .env -f docker-compose.infrastructure.yml up -d +``` + +--- + +## 4 · Launch Stella Ops 🚀 + +```bash +docker compose --env-file .env -f docker-compose.stella-ops.yml pull +docker compose --env-file .env -f docker-compose.stella-ops.yml up -d +``` + +*Point your browser at* **`https://:8443`** – the certificate is +self‑signed in the alpha. +Default credentials: **`admin / changeme`** (rotate immediately!). + +--- + +## 5 · Run a scan 🔍 + +```bash +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella-ops stella scan alpine:3.20 +``` + +* First scan downloads CVE feeds (\~ 50 MB). +* Warm scans finish in **≈ 5 s** on a 4‑vCPU host thanks to the Δ‑SBOM engine. + +--- + +## 6 · Reload or add a token later 🔄 + +```bash +# After adding STELLA_JWT to .env … +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella-ops stella jwt +``` + +*Anonymous mode* → **{{ quota_anon }} scans/day** +*Token mode* → **{{ quota_token }} scans/day** +At **10 % of the daily max** a polite reminder appears; after {{ quota_token }} the server applies a **soft 5 s back‑off** and may return **429 + Retry‑After** until the daily reset. + +--- + +## 7 · Typical next steps ➡️ + +| Task | Where to look | +| ---------------------------------------- | ------------------------------------------------------------------- | +| CI pipelines (GitHub / GitLab / Jenkins) | [`docs/ci/`](ci/) | +| Air‑gapped install | [Offline Update Kit](10_OFFLINE_KIT.md) | +| Feature overview | [20\_FEATURES.md](20_FEATURES.md) | +| Governance & licence | [`LICENSE.md`](LICENSE.md) • [`11_GOVERNANCE.md`](11_GOVERNANCE.md) | + +--- + +## 8 · Uninstall / cleanup 🧹 + +```bash +docker compose --env-file .env -f docker-compose.stella-ops.yml down -v +docker compose --env-file .env -f docker-compose.infrastructure.yml down -v +rm compose-*.yml compose-*.yml.sig .env +``` + +--- + +### Licence & provenance 📜 + +Stella Ops is **AGPL‑3.0‑or‑later**. Every release ships: + +* **Cosign‑signed** container images +* A full **SPDX 2.3** SBOM + +```bash +cosign verify \ + --key https://stella-ops.org/keys/cosign.pub \ + registry.stella-ops.org/stella-ops/stella-ops: +``` + +--- + +© 2025‑2026 Stella Ops – free / libre / open‑source. diff --git a/docs/03_VISION.md b/docs/03_VISION.md new file mode 100755 index 00000000..4508b0ea --- /dev/null +++ b/docs/03_VISION.md @@ -0,0 +1,99 @@ +#  3 · Product Vision — **Stella Ops** +*(v1.3 — 12 Jul 2025 · supersedes v1.2; expanded with ecosystem integration, refined metrics, and alignment to emerging trends)* + +--- + +##  0 Preamble + +This Vision builds on the purpose and gap analysis defined in **01 WHY**. +It paints a three‑year “north‑star” picture of success for the open‑source project and sets the measurable guard‑rails that every roadmap item must serve, while fostering ecosystem growth and adaptability to trends like SBOM mandates, AI‑assisted security **and transparent usage quotas**. + +--- + +##  1 North‑Star Vision Statement (2027) + +> *By mid‑2027, Stella Ops is the fastest, most‑trusted self‑hosted SBOM scanner. Developers expect vulnerability feedback in **five seconds or less**—even while the free tier enforces a transparent **{{ quota_token }} scans/day** limit with graceful waiting. The project thrives on a vibrant plug‑in marketplace, weekly community releases, transparent governance, and seamless integrations with major CI/CD ecosystems—while never breaking the five‑second promise.* + +--- + +##  2 Outcomes & Success Metrics + +| KPI (community‑centric) | Baseline Jul 2025 | Target Q2‑2026 | North‑Star 2027 | +| -------------------------------- | ----------------- | -------------- | --------------- | +| ⭐ Gitea / GitHub stars | 0 | 4 000 | 10 000 | +| Weekly active Docker pulls | 0 | 1 500 | 4 000 | +| P95 SBOM scan time (alpine) | 5 s | **≤ 5 s** | **≤ 4 s** | +| Free‑tier scan satisfaction* | n/a | ≥ 90 % | ≥ 95 % | +| First‑time‑contributor PRs / qtr | 0 | 15 | 30 | + +\*Measured via anonymous telemetry *opt‑in only*: ratio of successful scans to `429 QuotaExceeded` errors. + +--- + +##  3 Strategic Pillars + +1. **Speed First** – preserve the sub‑5 s P95 wall‑time; any feature that hurts it must ship behind a toggle or plug‑in. **Quota throttling must apply a soft 5 s delay first, so “speed first” remains true even at the limit.** +2. **Offline‑by‑Design** – every byte required to scan ships in public images; Internet access is optional. +3. **Modular Forever** – capabilities land as hot‑load plug‑ins; the monolith can split without rewrites. +4. **Community Ownership** – ADRs and governance decisions live in public; new maintainers elected by meritocracy. +5. **Zero‑Surprise Upgrades & Limits** – SemVer discipline; `main` is always installable; minor upgrades never break CI YAML **and free‑tier limits are clearly documented, with early UI warnings.** +6. **Ecosystem Harmony** – Prioritise integrations with popular OSS tools (e.g., Trivy extensions, BuildKit hooks) to lower adoption barriers. + +--- + +##  4 Road‑map Themes (18‑24 months) + +| Horizon | Theme | Example EPIC | +| ------------------ | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| **Q3‑2025** (3 mo) | **Core Stability & UX** | One‑command installer; dark‑mode UI; baseline SBOM scanning; **Free‑tier Quota Service ({{ quota_token }} scans/day, early banner, wait‑wall).** | +| 6–12 mo | *Extensibility* | Scan‑service micro‑split PoC; community plugin marketplace beta. | +| 12–18 mo | *Ecosystem* | Community plug‑in marketplace launch; integrations with Syft and Harbor. | +| 18–24 mo | *Resilience & Scale* | Redis Cluster auto‑sharding; AI‑assisted triage plugin framework. | + +*(Granular decomposition lives in 25_LEDGER.md.) + +--- + +##  5 Stakeholder Personas & Benefits + +| Persona | Core Benefit | +| --------------------- | ---------------------------------------------------------------- | +| Solo OSS maintainer | Laptop scans in **≤ 5 s**; zero cloud reliance. | +| CI Platform Engineer | Single‑binary backend + Redis; stable YAML integrations. | +| Security Auditor | AGPL code, traceable CVE sources, reproducible benchmarks. | +| Community Contributor | Plugin hooks and good‑first issues; merit‑based maintainer path. | +| Budget‑conscious Lead | Clear **{{ quota_token }} scans/day** allowance before upgrades are required. | + +(See **01 WHY §3** for detailed pain‑points & evidence.) + +--- + +##  6 Non‑Goals (2025‑2027) + +* Multi‑tenant SaaS offering. +* Automated “fix PR” generation. +* Proprietary compliance certifications (left to downstream distros). +* Windows **container** scanning (agents only). + +--- + +##  7 Review & Change Process + +* **Cadence:** product owner leads a public Vision review every **2 sprints (≈ 1 quarter)**. +* **Amendments:** material changes require PR labelled `type:vision` + two maintainer approvals. +* **Versioning:** bump patch for typo, minor for KPI tweak, major if North‑Star statement shifts. +* **Community Feedback:** Open GitHub Discussions for input; incorporate top‑voted suggestions quarterly. + +--- + +## 8 · Change Log + +| Version | Date | Note (high‑level) | +| ------- | ----------- | ----------------------------------------------------------------------------------------------------- | +| v1.4 | 14‑Jul‑2025 | First public revision reflecting quarterly roadmap & KPI baseline. | +| v1.3 | 12‑Jul‑2025 | Expanded ecosystem pillar, added metrics/integrations, refined non-goals, community persona/feedback. | +| v1.2 | 11‑Jul‑2025 | Restructured to link with WHY; merged principles into Strategic Pillars; added review §7 | +| v1.1 | 11‑Jul‑2025 | Original OSS‑only vision | +| v1.0 | 09‑Jul‑2025 | First public draft | + +*(End of Product Vision v1.3)* diff --git a/docs/04_FEATURE_MATRIX.md b/docs/04_FEATURE_MATRIX.md new file mode 100755 index 00000000..4e4efb70 --- /dev/null +++ b/docs/04_FEATURE_MATRIX.md @@ -0,0 +1,34 @@ +# 4 · Feature Matrix — **Stella Ops** +*(rev 2.0 · 14 Jul 2025)* + +| Category | Capability | Free Tier (≤ 333 scans / day) | Community Plug‑in | Commercial Add‑On | Notes / ETA | +| ---------------------- | ------------------------------------- | ----------------------------- | ----------------- | ------------------- | ------------------------------------------ | +| **SBOM Ingestion** | Trivy‑JSON, SPDX‑JSON, CycloneDX‑JSON | ✅ | — | — | Auto‑detect on upload | +| | **Delta‑SBOM Cache** | ✅ | — | — | Warm scans < 1 s | +| **Scanning** | CVE lookup via local DB | ✅ | — | — | Update job ships weekly feeds | +| | Licence‑risk detection | ⏳ (roadmap Q4‑2025) | — | — | SPDX licence list | +| **Policy Engine** | YAML rules | ✅ | — | — | In‑UI editor | +| | OPA / Rego | ⏳ (β Q1‑2026) | ✅ plug‑in | — | Plug‑in enables Rego | +| **Registry** | Anonymous internal registry | ✅ | — | — | `StellaOps.Registry` image | +| **Attestation** | Cosign signing | ⏳ (Q1‑2026) | — | — | Requires `StellaOpsAttestor` | +| | SLSA provenance v1.0 | — | — | ⏳ (commercial 2026) | Enterprise need | +| | Rekor transparency log | — | ✅ plug‑in | — | Air‑gap replica support | +| **Quota & Throttling** | {{ quota_token }} scans/day soft limit | ✅ | — | — | Yellow banner at 200, wait‑wall post‑limit | +| | Usage API (`/quota`) | ✅ | — | — | CI can poll remaining scans | +| **User Interface** | Dark / light mode | ✅ | — | — | Auto‑detect OS theme | +| | Additional locale (Cyrillic) | ✅ | — | — | Default if `Accept‑Language: bg` or any other | +| | Audit trail | ✅ | — | — | Mongo history | +| **Deployment** | Docker Compose bundle | ✅ | — | — | Single‑node | +| | Helm chart (K8s) | ✅ | — | — | Horizontal scaling | +| | High‑availability split services | — | — | ✅ (Add‑On) | HA Redis & Mongo | +| **Extensibility** | .NET hot‑load plug‑ins | ✅ | N/A | — | AGPL reference SDK | +| | Community plug‑in marketplace | — | ⏳ (β Q2‑2026) | — | Moderated listings | +| **Telemetry** | Opt‑in anonymous metrics | ✅ | — | — | Required for quota satisfaction KPI | +| **Quota & Tokens** | **Client‑JWT issuance** | ✅ (online 12 h token) | — | — | `/connect/token` | +| | **Offline Client‑JWT (30 d)** | ✅ via OUK | — | — | Refreshed monthly in OUK | + +> **Legend:** ✅ = Included ⏳ = Planned — = Not applicable +> Rows marked “Commercial Add‑On” are optional paid components shipping outside the AGPL‑core; everything else is FOSS. + +--- +*Last updated: 14 Jul 2025 (quota rev 2.0).* diff --git a/docs/05_ROADMAP.md b/docs/05_ROADMAP.md new file mode 100755 index 00000000..1a1402f3 --- /dev/null +++ b/docs/05_ROADMAP.md @@ -0,0 +1,6 @@ +# Road‑map + +Milestones are maintained on the project website. +👉  + +_This stub exists to satisfy historic links._ \ No newline at end of file diff --git a/docs/05_SYSTEM_REQUIREMENTS_SPEC.md b/docs/05_SYSTEM_REQUIREMENTS_SPEC.md new file mode 100755 index 00000000..ddaa6a26 --- /dev/null +++ b/docs/05_SYSTEM_REQUIREMENTS_SPEC.md @@ -0,0 +1,204 @@ +# SYSTEM REQUIREMENTS SPECIFICATION +Stella Ops · self‑hosted supply‑chain‑security platform + +> **Audience** – core maintainers and external contributors who need an +> authoritative checklist of *what* the software must do (functional +> requirements) and *how well* it must do it (non‑functional +> requirements). Implementation details belong in Module Specs +> or ADRs—**not here**. + +--- + +## 1 · Purpose & Scope + +This SRS defines everything the **v0.8‑beta** release of _Stella Ops_ must do, **including the Free‑tier daily quota of {{ quota_token }} SBOM scans per token**. +Scope includes core platform, CLI, UI, quota layer, and plug‑in host; commercial or closed‑source extensions are explicitly out‑of‑scope. + +--- + +## 2 · References + +* [02_WHY.md](02_WHY.md) – market gap & problem statement +* [03_VISION.md](03_VISION.md) – north‑star, KPIs, quarterly themes +* [07_HIGH_LEVEL_ARCHITECTURE.md](07_HIGH_LEVEL_ARCHITECTURE.md) – context & data flow diagrams +* [08_MODULE_SPECIFICATIONS.md](08_MODULE_SPECIFICATIONS.md) – component APIs & plug‑in contracts +* [09_API_CLI_REFERENCE.md](09_API_CLI_REFERENCE.md) – REST & CLI surface + +--- + +## 3 · Definitions & Acronyms + +| Term | Meaning | +|------|---------| +| **SBOM** | Software Bill of Materials | +| **Delta SBOM** | Partial SBOM covering only image layers not previously analysed | +| **Registry** | Anonymous, read‑only Docker Registry v2 hosted internally | +| **OPA** | Open Policy Agent (Rego policy engine) | +| **Muting Policy** | Rule that downgrades or ignores specific findings | +| **SLSA** | Supply‑chain Levels for Software Artifacts (provenance framework) | +| **Rekor** | Sigstore transparency log for signatures | + +--- + +## 4 · Overall System Description + +The platform consists of: + +* **Stella Ops Backend** – REST API, queue, policy engine, DB. +* **StellaOps.Registry** – internal container registry for agents. +* **Stella CLI** – extracts SBOMs; supports multi‑format & delta. +* **Zastava Agent** – enforcement hook for admission‑control scenarios. +* **Web UI** – React/Next.js SPA consuming backend APIs. +* **Plug‑ins** – hot‑load binaries extending scanners, attestations, etc. + +All services run in Docker Compose or Kubernetes with optional Internet +access. + +--- + +## 5 · Functional Requirements (FR) + +### 5.1 Core Scanning + +| ID | Requirement | Priority | Verification | +|----|-------------|----------|--------------| +| F‑1 | System SHALL ingest **Trivy‑JSON, SPDX‑JSON, CycloneDX‑JSON** files. | MUST | UT‑SBOM‑001 | +| F‑2 | System SHALL **auto‑detect** SBOM type when `sbomType` param omitted. | MUST | UT‑SBOM‑002 | +| F‑3 | System SHALL **cache analysed layers** and reuse them in subsequent scans. | MUST | IT‑CACHE‑001 | +| F‑4 | System SHALL **enforce a soft limit of {{ quota_token }} scans per token per UTC day**. | MUST | IT‑QUOTA‑001 | +| F‑4a | Remaining quota SHALL be **persisted in Redis** under key `quota::`. | MUST | UT‑QUOTA‑REDIS | +| F‑4b | Exhausted quota SHALL trigger **HTTP 429** with `Retry‑After` header (UTC midnight). | MUST | IT‑QUOTA‑002 | +| F‑4c | When quota is ≤ 40 % remaining, **UI banner** MUST turn yellow and show count‑down. | SHOULD | UI‑E2E‑005 | +| F‑4d | `/quota` endpoint SHALL return JSON `{"limit":{{ quota_token }} ,"remaining":N,"resetsAt":""}`. | SHOULD | API‑DOC‑003 | +| F‑5 | Policy engine SHALL evaluate **YAML rules** against scan results. | MUST | UT‑POL‑001 | +| F‑6 | Hot‑pluggable .NET plug‑ins SHALL be loadable **without service restart**. | MUST | IT‑PLUGIN‑001 | +| F‑7 | CLI (`stella scan`) SHOULD exit **non‑zero** when CVSS≥7 vulnerabilities found. | SHOULD | CL‑INT‑003 | +| *(… all previously documented F‑8 – F‑12 rows retained unchanged …)* | + + +### 5.2 Internal Docker Repository + +| Ref | Requirement | +|-----|-------------| +| **FR‑REPO‑1** | Platform SHALL include **StellaOps.Registry** exposing Docker Registry v2 API (ports 5000/443). | +| **FR‑REPO‑2** | Registry SHALL allow anonymous, *read‑only* pulls for at least three images:
• `stella/sbom‑builder`
• `stella/cli`
• `stella/zastava`. | +| **FR‑REPO‑3** | Registry MAY enable optional basic‑auth without code changes. | + +### 5.3 SBOM Generation & Handling + +| Ref | Requirement | +|-----|-------------| +| **FR‑SBOM‑1** | SBOM builder SHALL produce Trivy‑JSON **and** at least one additional format: SPDX‑JSON and CycloneDX‑JSON. | +| **FR‑SBOM‑2** | For every generated SBOM, builder SHALL create a side‑car file `.sbom.type` containing the format identifier. | +| **FR‑SBOM‑3** | Stella CLI SHALL read the `.sbom.type` file and include `sbomType` parameter when uploading. | +| **FR‑SBOM‑4** | Backend SHALL auto‑detect SBOM type when parameter is missing. | +| **FR‑SBOM‑5** | UI Settings SHALL expose a dropdown to select default SBOM format (system‑wide fallback). | + +#### 5.3.1 Delta SBOM (layer reuse) + +| Ref | Requirement | +|-----|-------------| +| **FR‑DELTA‑1** | Builder SHALL compute SHA256 digests of each image layer and POST array to `/layers/missing`; response time ≤ 20 ms (P95). | +| **FR‑DELTA‑2** | Builder SHALL generate SBOM **only** for layers returned as “missing”. | +| **FR‑DELTA‑3** | End‑to‑end warm scan time (image differing by ≤ 2 layers) SHALL be ≤ 1 s (P95). | + +### 5.4 Policy as Code (Muting & Expiration) + +| Ref | Requirement | +|-----|-------------| +| **FR‑POLICY‑1** | Backend SHALL store policies as YAML by default, convertible to Rego for advanced use‑cases. | +| **FR‑POLICY‑2** | Each policy change SHALL create an immutable history record (timestamp, actor, diff). | +| **FR‑POLICY‑3** | REST endpoints `/policy/import`, `/policy/export`, `/policy/validate` SHALL accept YAML or Rego payloads. | +| **FR‑POLICY‑4** | Web UI Policies tab SHALL provide Monaco editor with linting for YAML and Rego. | +| **FR‑POLICY‑5** | **StellaOps.MutePolicies** module SHALL expose CLI `stella policies apply --file scan‑policy.yaml`. | + +### 5.5 SLSA Attestations & Rekor (TODO > 6 mo) + +| Ref | Requirement | +|-----|-------------| +| **FR‑SLSA‑1** | **TODO** – Generate provenance in SLSA‑Provenance v0.2 for each SBOM. | +| **FR‑REKOR‑1** | **TODO** – Sign SBOM hashes and upload to local Rekor mirror; verify during scan. | + +### 5.6 CLI & API Interface + +| Ref | Requirement | +|-----|-------------| +| **FR‑CLI‑1** | CLI `stella scan` SHALL accept `--sbom-type {trivy,spdx,cyclonedx,auto}`. | +| **FR‑API‑1** | API `/scan` SHALL accept `sbomType` query/body field (optional). | +| **FR‑API‑2** | API `/layers/missing` SHALL accept JSON array of digests and return JSON array of missing digests. | + +--- + +## 6 · Non‑Functional Requirements (NFR) + +| Ref | Category | Requirement | +|-----|----------|-------------| +| **NFR‑PERF‑1** | Performance | P95 cold scan ≤ 5 s; warm ≤ 1 s (see **FR‑DELTA‑3**). | +| **NFR‑PERF‑2** | Throughput | System shall sustain 60 concurrent scans on 8‑core node without queue depth >10. | +| **NFR‑AVAIL‑1** | Availability | All services shall start offline; any Internet call must be optional. | +| **NFR‑SCAL‑1** | Scalability | Horizontal scaling via Kubernetes replicas for backend, Redis Sentinel, Mongo replica set. | +| **NFR‑SEC‑1** | Security | All inter‑service traffic shall use TLS or localhost sockets. | +| **NFR‑COMP‑1** | Compatibility | Platform shall run on x86‑64 Linux kernel ≥ 5.10; Windows agents (TODO > 6 mo) must support Server 2019+. | +| **NFR‑I18N‑1** | Internationalisation | UI must support EN and at least one additional locale (Cyrillic). | +| **NFR‑OBS‑1** | Observability | Export Prometheus metrics for scan duration, queue length, policy eval duration. | + +--- + +## 7 Acceptance Criteria + +1. Issue {{ quota_token }} `/scan` calls; next returns random slow down and `Retry‑After`. +2. Redis failure during test → API returns **0 remaining** & warns in logs. +3. UI banner activates at 133 remaining; clears next UTC midnight. + +--- +## 8 · System Interfaces + +### 8.1 External APIs + +*(This is the complete original table, plus new `/quota` row.)* + +| Path | Method | Auth | Quota | Description | +|------|--------|------|-------|-------------| +| `/scan` | POST | Bearer | ✅ | Submit SBOM or `imageRef` for scanning. | +| `/quota` | GET | Bearer | ❌ | Return remaining quota for current token. | +| `/policy/rules` | GET/PUT | Bearer+RBAC | ❌ | CRUD YAML or Rego policies. | +| `/plugins` | POST/GET | Bearer+Admin | ❌ | Upload or list plug‑ins. | + +```bash +GET /quota +Authorization: Bearer + +200 OK +{ +"limit": {{ quota_token }}, +"remaining": 121, +"resetsAt": "2025-07-14T23:59:59Z" +} +``` + +## 9 · Assumptions & Constraints + +* Hardware reference: 8 vCPU, 8 GB RAM, NVMe SSD. +* Mongo DB and Redis run co‑located unless horizontal scaling enabled. +* All docker images tagged `latest` are immutable (CI process locks digests). +* Rego evaluation runs in embedded OPA Go‑library (no external binary). + +--- + +## 10 · Future Work (Beyond 12 Months) + +* Rekor transparency log cross‑cluster replication. +* AI‑assisted false‑positive triage plug‑in. +* Cluster‑wide injection for live runtime scanning. + +--- + +## 11 · Revision History + +| Version | Date | Notes | +|---------|------|-------| +| **v1.2** | 11‑Jul‑2025 | Commercial references removed; plug‑in contract (§ 3.3) and new NFR categories added; added User Classes & Traceability. | +| v1.1 | 11‑Jul‑2025 | Split out RU‑specific items; OSS scope | +| v1.0 | 09‑Jul‑2025 | Original unified SRS | + +*(End of System Requirements Specification v1.2‑core)* diff --git a/docs/07_HIGH_LEVEL_ARCHITECTURE.md b/docs/07_HIGH_LEVEL_ARCHITECTURE.md new file mode 100755 index 00000000..fd8d5250 --- /dev/null +++ b/docs/07_HIGH_LEVEL_ARCHITECTURE.md @@ -0,0 +1,388 @@ +# 7 · High‑Level Architecture — **Stella Ops** + +--- + +## 0 Purpose & Scope + +Give contributors, DevOps engineers and auditors a **complete yet readable map** of the Core: + +* Major runtime components and message paths. +* Where plug‑ins, CLI helpers and runtime agents attach. +* Technology choices that enable the sub‑5 second SBOM goal. +* Typical operational scenarios (pipeline scan, mute, nightly re‑scan, etc.). + +Anything enterprise‑only (signed PDF, custom/regulated TLS, LDAP, enforcement) **must arrive as a plug‑in**; the Core never hard‑codes those concerns. +--- +## 1 Component Overview + +| # | Component | Responsibility | +|---|-----------|---------------| +| 1 | **API Gateway** | REST endpoints (`/scan`, `/quota`, **`/token/offline`**); token auth; quota enforcement | +| 2 | **Scan Service** | SBOM parsing, Delta‑SBOM cache, vulnerability lookup | +| 3 | **Policy Engine** | YAML / (optional) Rego rule evaluation; verdict assembly | +| 4 | **Quota Service** | Per‑token counters; **333 scans/day**; waits & HTTP 429 | +| 5 | **Client‑JWT Issuer** | Issues 30‑day offline tokens; bundles them into OUK | +| 5 | **Registry** | Anonymous internal Docker registry for agents, SBOM uploads | +| 6 | **Web UI** | React/Blazor SPA; dashboards, policy editor, quota banner | +| 7 | **Data Stores** | **Redis** (cache, quota) & **MongoDB** (SBOMs, findings, audit) | +| 8 | **Plugin Host** | Hot‑load .NET DLLs; isolates community plug‑ins | +| 9 | **Agents** | `sbom‑builder`, `Stella CLI` scanner CLI, future `StellaOpsAttestor` | + + +```mermaid +flowchart TD + subgraph "External Actors" + DEV["Developer / DevSecOps / Manager"] + CI["CI/CD Pipeline (e.g., Stella CLI)"] + K8S["Kubernetes Cluster (e.g., Zastava Agent)"] + end + + subgraph "Stella Ops Runtime" + subgraph "Core Services" + CORE["Stella Core
(REST + gRPC APIs, Orchestration)"] + REDIS[("Redis
(Cache, Queues, Trivy DB Mirror)")] + MONGO[("MongoDB
(Optional: Long-term Storage)")] + POL["Mute Policies
(OPA & YAML Evaluator)"] + REG["StellaOps Registry
(Docker Registry v2)"] + ATT["StellaOps Attestor
(SLSA + Rekor)"] + end + + subgraph "Agents & Builders" + SB["SBOM Builder
(Go Binary: Extracts Layers, Generates SBOMs)"] + SA["Stella CLI
(Pipeline Helper: Invokes Builder, Triggers Scans)"] + ZA["Zastava Agent
(K8s Webhook: Enforces Policies, Inventories Containers)"] + end + + subgraph "Scanners & UI" + TRIVY["Trivy Scanner
(Plugin Container: Vulnerability Scanning)"] + UI["Web UI
(Vue3 + Tailwind: Dashboards, Policy Editor)"] + CLI["Stella CLI
(CLI Helper: Triggers Scans, Mutes)"] + end + end + + DEV -->|Browses Findings, Mutes CVEs| UI + DEV -->|Triggers Scans| CLI + CI -->|Generates SBOM, Calls /scan| SA + K8S -->|Inventories Containers, Enforces Gates| ZA + + UI -- "REST" --> CORE + CLI -- "REST/gRPC" --> CORE + SA -->|Scan Requests| CORE + SB -->|Uploads SBOMs| CORE + ZA -->|Policy Gates| CORE + + CORE -- "Queues, Caches" --> REDIS + CORE -- "Persists Data" --> MONGO + CORE -->|Evaluates Policies| POL + CORE -->|Attests Provenance| ATT + CORE -->|Scans Vulnerabilities| TRIVY + + SB -- "Pulls Images" --> REG + SA -- "Pulls Images" --> REG + ZA -- "Pulls Images" --> REG + + style DEV fill:#f9f,stroke:#333 + style CI fill:#f9f,stroke:#333 + style K8S fill:#f9f,stroke:#333 + style CORE fill:#ddf,stroke:#333 + style REDIS fill:#fdd,stroke:#333 + style MONGO fill:#fdd,stroke:#333 + style POL fill:#dfd,stroke:#333 + style REG fill:#dfd,stroke:#333 + style ATT fill:#dfd,stroke:#333 + style SB fill:#fdf,stroke:#333 + style SA fill:#fdf,stroke:#333 + style ZA fill:#fdf,stroke:#333 + style TRIVY fill:#ffd,stroke:#333 + style UI fill:#ffd,stroke:#333 + style CLI fill:#ffd,stroke:#333 +``` + +* **Developer / DevSecOps / Manager** – browses findings, mutes CVEs, triggers scans. +* **Stella CLI** – generates SBOMs and calls `/scan` during CI. +* **Zastava Agent** – inventories live containers; Core ships it in *passive* mode only (no kill). + +### 1.1 Client‑JWT Lifecycle (offline aware) + +1. **Online instance** – user signs in → `/connect/token` issues JWT valid 12 h. +2. **Offline instance** – JWT with `exp ≈ 30 days` ships in OUK; backend + **re‑signs** and stores it during import. +3. Tokens embed a `tier` claim (“Free”) and `maxScansPerDay: 333`. +4. On expiry the UI surfaces a red toast **7 days** in advance. + +--- + +## 2 · Component Responsibilities (runtime view) + +| Component | Core Responsibility | Implementation Highlights | +| -------------------------- | ---------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | +| **Stella Core** | Orchestrates scans, persists SBOM blobs, serves REST/gRPC APIs, fans out jobs to scanners & policy engine. | .NET {{ dotnet }}, CQRS, Redis Streams; pluggable runner interfaces. | +| **SBOM Builder** | Extracts image layers, queries Core for *missing* layers, generates SBOMs (multi‑format), uploads blobs. | Go binary; wraps Trivy & Syft libs. | +| **Stella CLI** | Pipeline‑side helper; invokes Builder, triggers scan, streams progress back to CI/CD. | Static musl build. | +| **Zastava Agent** | K8s admission webhook enforcing policy verdicts before Pod creation. | Rust for sub‑10 ms latencies. | +| **UI** | Angular 17 SPA for dashboards, settings, policy editor. | Tailwind CSS; Webpack module federation (future). | +| **Redis** | Cache, queue, Trivy‑DB mirror, layer diffing. | Single instance or Sentinel. | +| **MongoDB** (opt.) | Long‑term SBOM & policy audit storage (> 180 days). | Optional; enabled via flag. | +| **StellaOps.Registry** | Anonymous read‑only Docker v2 registry with optional Cosign verification. | `registry :2` behind nginx reverse proxy. | +| **StellaOps.MutePolicies** | YAML/Rego evaluator, policy version store, `/policy/*` API. | Embeds OPA‑WASM; falls back to `opa exec`. | +| **StellaOpsAttestor** | Generate SLSA provenance & Rekor signatures; verify on demand. | Side‑car container; DSSE + Rekor CLI. | + +All cross‑component calls use dependency‑injected interfaces—no +intra‑component reach‑ins. + +--- + +## 3 · Principal Backend Modules & Plug‑in Hooks + +| Namespace | Responsibility | Built‑in Tech / Default | Plug‑in Contract | +| --------------- | -------------------------------------------------- | ----------------------- | ------------------------------------------------- | +| `configuration` | Parse env/JSON, health‑check endpoint | .NET {{ dotnet }} Options | `IConfigValidator` | +| `identity` | Embedded OAuth2/OIDC (OpenIddict 6) | MIT OpenIddict | `IIdentityProvider` for LDAP/SAML/JWT gateway | +| `pluginloader` | Discover DLLs, SemVer gate, optional Cosign verify | Reflection + Cosign | `IPluginLifecycleHook` for telemetry | +| `scanning` | SBOM‑ & image‑flow orchestration; runner pool | Trivy CLI (default) | `IScannerRunner` – e.g., Grype, Copacetic, Clair | +| `feedser` (vulnerability ingest/merge/export service) | Nightly NVD merge & feed enrichment | Hangfire job | drop-in `*.Schedule.dll` for OSV, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds | +| `tls` | TLS provider abstraction | OpenSSL | `ITlsProvider` for custom suites (incl. **SM2**, where law or security requires it) | +| `reporting` | Render HTML/PDF reports | RazorLight | `IReportRenderer` | +| `ui` | Angular SPA & i18n | Angular {{ angular }} | new locales via `/locales/{lang}.json` | +| `scheduling` | Cron + retries | Hangfire | any recurrent job via `*.Schedule.dll` | + +```mermaid +classDiagram + class configuration + class identity + class pluginloader + class scanning + class feedser + class tls + class reporting + class ui + class scheduling + + class AllModules + + configuration ..> identity : Uses + identity ..> pluginloader : Authenticates Plugins + pluginloader ..> scanning : Loads Scanner Runners + scanning ..> feedser : Triggers Feed Merges + tls ..> AllModules : Provides TLS Abstraction + reporting ..> ui : Renders Reports for UI + scheduling ..> feedser : Schedules Nightly Jobs + + note for scanning "Pluggable: ISScannerRunner
e.g., Trivy, Grype" + note for feedser "Pluggable: *.Schedule.dll
e.g., OSV, GHSA Feeds" + note for identity "Pluggable: IIdentityProvider
e.g., LDAP, SAML" + note for reporting "Pluggable: IReportRenderer
e.g., Custom PDF" +``` + +**When remaining = 0:** +API returns `429 Too Many Requests`, `Retry‑After: ` (sequence omitted for brevity). + +--- + +## 4 · Data Flows + +### 4.1 SBOM‑First (≤ 5 s P95) + +Builder produces SBOM locally, so Core never touches the Docker +socket. +Trivy path hits ≤ 5 s on alpine:3.19 with warmed DB. +Image‑unpack fallback stays ≤ 10 s for 200 MB images. + +```mermaid +sequenceDiagram + participant CI as CI/CD Pipeline (Stella CLI) + participant SB as SBOM Builder + participant CORE as Stella Core + participant REDIS as Redis Queue + participant RUN as Scanner Runner (e.g., Trivy) + participant POL as Policy Evaluator + + CI->>SB: Invoke SBOM Generation + SB->>CORE: Check Missing Layers (/layers/missing) + CORE->>REDIS: Query Layer Diff (SDIFF) + REDIS-->>CORE: Missing Layers List + CORE-->>SB: Return Missing Layers + SB->>SB: Generate Delta SBOM + SB->>CORE: Upload SBOM Blob (POST /scan(sbom)) + CORE->>REDIS: Enqueue Scan Job + REDIS->>RUN: Fan Out to Runner + RUN->>RUN: Perform Vulnerability Scan + RUN-->>CORE: Return Scan Results + CORE->>POL: Evaluate Mute Policies + POL-->>CORE: Policy Verdict + CORE-->>CI: JSON Verdict & Progress Stream + Note over CORE,CI: Achieves ≤5s P95 with Warmed DB +``` + +### 4.2 Delta SBOM + +Builder collects layer digests. +`POST /layers/missing` → Redis SDIFF → missing layer list (< 20 ms). +SBOM generated only for those layers and uploaded. + +### 4.3 Feedser Harvest & Export + +```mermaid +sequenceDiagram + participant SCHED as Feedser Scheduler + participant CONN as Source Connector Plug-in + participant FEEDSER as Feedser Core + participant MONGO as MongoDB (Canonical Advisories) + participant EXPORT as Exporter (JSON / Trivy DB) + participant ART as Artifact Store / Offline Kit + + SCHED->>CONN: Trigger window (init/resume) + CONN->>CONN: Fetch source documents + metadata + CONN->>FEEDSER: Submit raw document for parsing + FEEDSER->>FEEDSER: Parse & normalize to DTO + FEEDSER->>FEEDSER: Merge & deduplicate canonical advisory + FEEDSER->>MONGO: Write advisory, provenance, merge_event + FEEDSER->>EXPORT: Queue export delta request + EXPORT->>MONGO: Read canonical snapshot/deltas + EXPORT->>EXPORT: Build deterministic JSON & Trivy DB artifacts + EXPORT->>ART: Publish artifacts / Offline Kit bundle + ART-->>FEEDSER: Record export state + digests +``` + +### 4.4 Identity & Auth Flow + +OpenIddict issues JWTs via client‑credentials or password grant. +An IIdentityProvider plug‑in can delegate to LDAP, SAML or external OIDC +without Core changes. +--- +## 5 · Runtime Helpers + +| Helper | Form | Purpose | Extensible Bits | +|-----------|---------------------------------------|--------------------------------------------------------------------|-------------------------------------------| +| **Stella CLI** | Distroless CLI | Generates SBOM, calls `/scan`, honours threshold flag | `--engine`, `--pdf-out` piped to plug‑ins | +| **Zastava** | Static Go binary / DaemonSet | Watches Docker/CRI‑O events; uploads SBOMs; can enforce gate | Policy plug‑in could alter thresholds | + +--- + +## 6 · Persistence & Cache Strategy + +| Store | Primary Use | Why chosen | +|----------------|-----------------------------------------------|--------------------------------| +| **MongoDB** | Feedser canonical advisories, merge events, export state | Deterministic canonical store with flexible schema | +| **Redis 7** | CLI quotas, short-lived job scheduling, layer diff cache | Sub-1 ms P99 latency for hot-path coordination | +| **Local tmpfs**| Trivy layer cache (`/var/cache/trivy`) | Keeps disk I/O off hot path | + +```mermaid +flowchart LR + subgraph "Persistence Layers" + REDIS[(Redis: Quotas & Short-lived Queues
Sub-1ms P99)] + MONGO[(MongoDB: Canonical Advisories
Merge Events & Export State)] + TMPFS[(Local tmpfs: Trivy Layer Cache
Low I/O Overhead)] + end + + CORE["Stella Core"] -- Queues & SBOM Cache --> REDIS + CORE -- Long-term Storage --> MONGO + TRIVY["Trivy Scanner"] -- Layer Unpack Cache --> TMPFS + + style REDIS fill:#fdd,stroke:#333 + style MONGO fill:#dfd,stroke:#333 + style TMPFS fill:#ffd,stroke:#333 +``` + +--- + +## 7 · Typical Scenarios + +| # | Flow | Steps | +|---------|----------------------------|-------------------------------------------------------------------------------------------------| +| **S‑1** | Pipeline Scan & Alert | Stella CLI → SBOM → `/scan` → policy verdict → CI exit code & link to *Scan Detail* | +| **S‑2** | Mute Noisy CVE | Dev toggles **Mute** in UI → rule stored in Redis → next build passes | +| **S‑3** | Nightly Re‑scan | `SbomNightly.Schedule` re‑queues SBOMs (mask‑filter) → dashboard highlights new Criticals | +| **S‑4** | Feed Update Cycle | `Feedser (vulnerability ingest/merge/export service)` refreshes feeds → UI *Feed Age* tile turns green | +| **S‑5** | Custom Report Generation | Plug‑in registers `IReportRenderer` → `/report/custom/{digest}` → CI downloads artifact | + +```mermaid +sequenceDiagram + participant DEV as Developer + participant UI as Web UI + participant CORE as Stella Core + participant REDIS as Redis + participant RUN as Scanner Runner + + DEV->>UI: Toggle Mute for CVE + UI->>CORE: Update Mute Rule (POST /policy/mute) + CORE->>REDIS: Store Mute Policy + Note over CORE,REDIS: YAML/Rego Evaluator Updates + + alt Next Pipeline Build + CI->>CORE: Trigger Scan (POST /scan) + CORE->>RUN: Enqueue & Scan + RUN-->>CORE: Raw Findings + CORE->>REDIS: Apply Mute Policies + REDIS-->>CORE: Filtered Verdict (Passes) + CORE-->>CI: Success Exit Code + end +``` + +```mermaid +sequenceDiagram + participant CRON as SbomNightly.Schedule + participant CORE as Stella Core + participant REDIS as Redis Queue + participant RUN as Scanner Runner + participant UI as Dashboard + + CRON->>CORE: Re-queue SBOMs (Mask-Filter) + CORE->>REDIS: Enqueue Filtered Jobs + REDIS->>RUN: Fan Out to Runners + RUN-->>CORE: New Scan Results + CORE->>UI: Highlight New Criticals + Note over CORE,UI: Focus on Changes Since Last Scan +``` +--- + +## 8 · UI Fast Facts + +* **Stack** – Angular 17 + Vite dev server; Tailwind CSS. +* **State** – Signals + RxJS for live scan progress. +* **i18n / l10n** – JSON bundles served from `/locales/{lang}.json`. +* **Module Structure** – Lazy‑loaded feature modules (`dashboard`, `scans`, `settings`); runtime route injection by UI plug‑ins (road‑map Q2‑2026). + +--- + +## 9 · Cross‑Cutting Concerns + +* **Security** – containers run non‑root, `CAP_DROP:ALL`, read‑only FS, hardened seccomp profiles. +* **Observability** – Serilog JSON, OpenTelemetry OTLP exporter, Prometheus `/metrics`. +* **Upgrade Policy** – `/api/v1` endpoints & CLI flags stable across a minor; breaking changes bump major. + +--- + +## 10 · Performance & Scalability + +| Scenario | P95 target | Bottleneck | Mitigation | +|-----------------|-----------:|-----------------|-------------------------------------------------| +| SBOM‑first | ≤ 5 s | Redis queue | More CPU, increase `ScannerPool.Workers` | +| Image‑unpack | ≤ 10 s | Layer unpack | Prefer SBOM path, warm Docker cache | +| High concurrency| 40 rps | Runner CPU | Scale Core replicas + side‑car scanner services | + +--- + +## 11 · Future Architectural Anchors + +* **ScanService micro‑split (gRPC)** – isolate heavy runners for large clusters. +* **UI route plug‑ins** – dynamic Angular module loader (road‑map Q2‑2026). +* **Redis Cluster** – transparently sharded cache once sustained > 100 rps. + +--- + +## 12 · Assumptions & Trade‑offs + +Requires Docker/CRI‑O runtime; .NET 9 available on hosts; Windows containers are out‑of‑scope this cycle. +Embedded auth simplifies deployment but may need plug‑ins for enterprise IdPs. +Speed is prioritised over exhaustive feature parity with heavyweight commercial scanners. + +--- + +## 13 · References & Further Reading + +* **C4 Model** – +* **.NET Architecture Guides** – +* **OSS Examples** – Kubernetes Architecture docs, Prometheus design papers, Backstage. + +*(End of High‑Level Architecture v2.2)* diff --git a/docs/08_MODULE_SPECIFICATIONS.md b/docs/08_MODULE_SPECIFICATIONS.md new file mode 100755 index 00000000..53497ddd --- /dev/null +++ b/docs/08_MODULE_SPECIFICATIONS.md @@ -0,0 +1,201 @@ +# 8 · Detailed Module Specifications — **Stella Ops Feedser** +_This document describes the Feedser service, its supporting libraries, connectors, exporters, and test assets that live in the OSS repository._ + +--- + +## 0 Scope + +Feedser is the vulnerability ingest/merge/export subsystem of Stella Ops. It +fetches primary advisories, normalizes and deduplicates them into MongoDB, and +produces deterministic JSON and Trivy DB exports. This document lists the +projects that make up that workflow, the extension points they expose, and the +artefacts they ship. + +--- + +## 1 Repository layout (current) + +```text +src/ + ├─ Directory.Build.props / Directory.Build.targets + ├─ StellaOps.Plugin/ + ├─ StellaOps.Feedser.Core/ + ├─ StellaOps.Feedser.Core.Tests/ + ├─ StellaOps.Feedser.Models/ (+ .Tests/) + ├─ StellaOps.Feedser.Normalization/ (+ .Tests/) + ├─ StellaOps.Feedser.Merge/ (+ .Tests/) + ├─ StellaOps.Feedser.Storage.Mongo/ (+ .Tests/) + ├─ StellaOps.Feedser.Exporter.Json/ (+ .Tests/) + ├─ StellaOps.Feedser.Exporter.TrivyDb/ (+ .Tests/) + ├─ StellaOps.Feedser.Source.* / StellaOps.Feedser.Source.*.Tests/ + ├─ StellaOps.Feedser.Testing/ + ├─ StellaOps.Feedser.Tests.Shared/ + ├─ StellaOps.Feedser.WebService/ (+ .Tests/) + ├─ PluginBinaries/ + └─ StellaOps.Feedser.sln +``` + +Each folder is a .NET project (or set of projects) referenced by +`StellaOps.Feedser.sln`. Build assets are shared through the root +`Directory.Build.props/targets` so conventions stay consistent. + +--- + +## 2 Shared libraries + +| Project | Purpose | Key extension points | +|---------|---------|----------------------| +| `StellaOps.Plugin` | Base contracts for connectors, exporters, and DI routines plus Cosign validation helpers. | `IFeedConnector`, `IExporterPlugin`, `IDependencyInjectionRoutine` | +| `StellaOps.DependencyInjection` | Composable service registrations for Feedser and plug-ins. | `IDependencyInjectionRoutine` discovery | +| `StellaOps.Feedser.Testing` | Common fixtures, builders, and harnesses for integration/unit tests. | `FeedserMongoFixture`, test builders | +| `StellaOps.Feedser.Tests.Shared` | Shared assembly metadata and fixtures wired in via `Directory.Build.props`. | Test assembly references | + +--- + +## 3 Core projects + +| Project | Responsibility | Extensibility | +|---------|----------------|---------------| +| `StellaOps.Feedser.WebService` | ASP.NET Core minimal API hosting Feedser jobs, status endpoints, and scheduler. | DI-based plug-in discovery; configuration binding | +| `StellaOps.Feedser.Core` | Job orchestration, connector pipelines, merge workflows, export coordination. | `IFeedConnector`, `IExportJob`, deterministic merge policies | +| `StellaOps.Feedser.Models` | Canonical advisory DTOs and enums persisted in MongoDB and exported artefacts. | Partial classes for source-specific metadata | +| `StellaOps.Feedser.Normalization` | Version comparison, CVSS normalization, text utilities for canonicalization. | Helpers consumed by connectors/merge | +| `StellaOps.Feedser.Merge` | Precedence evaluation, alias graph maintenance, merge-event hashing. | Policy extensions via DI | +| `StellaOps.Feedser.Storage.Mongo` | Repository layer for documents, DTOs, advisories, merge events, export state. | Connection string/config via options | +| `StellaOps.Feedser.Exporter.Json` | Deterministic vuln-list JSON export pipeline. | Dependency injection for storage + plugin to host | +| `StellaOps.Feedser.Exporter.TrivyDb` | Builds Trivy DB artefacts from canonical advisories. | Optional ORAS push routines | + +### 3.1 StellaOps.Feedser.WebService + +* Hosts minimal API endpoints (`/health`, `/status`, `/jobs`). +* Runs the scheduler that triggers connectors and exporters according to + configured windows. +* Applies dependency-injection routines from `PluginBinaries/` at startup only + (restart-time plug-ins). + +### 3.2 StellaOps.Feedser.Core + +* Defines job primitives (fetch, parse, map, merge, export) used by connectors. +* Coordinates deterministic merge flows and writes `merge_event` documents. +* Provides telemetry/log scopes consumed by WebService and exporters. + +### 3.3 StellaOps.Feedser.Storage.Mongo + +* Persists raw documents, DTO records, canonical advisories, aliases, affected + packages, references, merge events, export state, and job leases. +* Exposes repository helpers for exporters to stream full/delta snapshots. + +### 3.4 StellaOps.Feedser.Exporter.* + +* `Exporter.Json` mirrors the Aqua vuln-list tree with canonical ordering. +* `Exporter.TrivyDb` builds Trivy DB Bolt archives and optional OCI bundles. +* Both exporters honour deterministic hashing and respect export cursors. + +--- + +## 4 Source connectors + +Connectors live under `StellaOps.Feedser.Source.*` and conform to the interfaces +in `StellaOps.Plugin`. + +| Family | Project(s) | Notes | +|--------|------------|-------| +| Distro PSIRTs | `StellaOps.Feedser.Source.Distro.*` | Debian, Red Hat, SUSE, Ubuntu connectors with NEVRA/EVR helpers. | +| Vendor PSIRTs | `StellaOps.Feedser.Source.Vndr.*` | Adobe, Apple, Cisco, Chromium, Microsoft, Oracle, VMware. | +| Regional CERTs | `StellaOps.Feedser.Source.Cert*`, `Source.Ru.*`, `Source.Ics.*`, `Source.Kisa` | Provide enrichment metadata while preserving vendor precedence. | +| OSS ecosystems | `StellaOps.Feedser.Source.Ghsa`, `Source.Osv`, `Source.Cve`, `Source.Kev`, `Source.Acsc`, `Source.Cccs`, `Source.Jvn` | Emit SemVer/alias-rich advisories. | + +Each connector ships fixtures/tests under the matching `*.Tests` project. + +--- + +## 5 · Module Details + +> _Focus on the Feedser-specific services that replace the legacy FeedMerge cron._ + +### 5.1 Feedser.Core + +* Owns the fetch → parse → merge → export job pipeline and enforces deterministic + merge hashes (`merge_event`). +* Provides `JobSchedulerBuilder`, job coordinator, and telemetry scopes consumed + by the WebService and exporters. + +### 5.2 Feedser.Storage.Mongo + +* Bootstrapper creates collections/indexes (documents, dto, advisory, alias, + affected, merge_event, export_state, jobs, locks). +* Repository APIs surface full/delta advisory reads for exporters, plus + SourceState and job lease persistence. + +### 5.3 Feedser.Exporter.Json / Feedser.Exporter.TrivyDb + +* JSON exporter mirrors vuln-list layout with per-file digests and manifest. +* Trivy DB exporter shells or native-builds Bolt archives, optionally pushes OCI + layers, and records export cursors. + +### 5.4 Feedser.WebService + +* Minimal API host exposing `/health`, `/ready`, `/jobs` and wiring telemetry. +* Loads restart-time plug-ins from `PluginBinaries/`, executes Mongo bootstrap, + and registers built-in connectors/exporters with the scheduler. + +### 5.5 Plugin host & DI bridge + +* `StellaOps.Plugin` + `StellaOps.DependencyInjection` provide the contracts and + helper routines for connectors/exporters to integrate with the WebService. + +--- + +## 6 · Plug-ins & Agents + +* **Plug-in discovery** – restart-only; the WebService enumerates + `PluginBinaries/` (or configured directories) and executes the contained + `IDependencyInjectionRoutine` implementations. +* **Connector/exporter packages** – each source/exporter can ship as a plug-in + assembly with its own options and HttpClient configuration, keeping the core + image minimal. +* **Stella CLI (agent)** – triggers feed-related jobs (`stella db fetch/merge/export`) + and consumes the exported JSON/Trivy DB artefacts, aligning with the SBOM-first + workflow described in `AGENTS.md`. +* **Offline Kit** – bundles Feedser plug-ins, JSON tree, Trivy DB, and export + manifests so air-gapped sites can load the latest vulnerability data without + outbound connectivity. + +--- + +## 7 · Docker & Distribution Artefacts + +| Artefact | Path / Identifier | Notes | +|----------|-------------------|-------| +| Feedser WebService image | `containers/feedser/Dockerfile` (built via CI) | Self-contained ASP.NET runtime hosting scheduler/endpoints. | +| Plugin bundle | `PluginBinaries/` | Mounted or baked-in assemblies for connectors/exporters. | +| Offline Kit tarball | Produced by CI release pipeline | Contains JSON tree, Trivy DB OCI layout, export manifest, and plug-ins. | +| Local dev compose | `scripts/` + future compose overlays | Developers can run MongoDB, Redis (optional), and WebService locally. | + +--- + +## 8 · Performance Budget + +| Scenario | Budget | Source | +|----------|--------|--------| +| Advisory upsert (large advisory) | ≤ 500 ms/advisory | `AdvisoryStorePerformanceTests` (Mongo) | +| Advisory fetch (`GetRecent`) | ≤ 200 ms/advisory | Same performance test harness | +| Advisory point lookup (`Find`) | ≤ 200 ms/advisory | Same performance test harness | +| Bulk upsert/fetch cycle | ≤ 28 s total for 30 large advisories | Same performance test harness | +| Feedser job scheduling | Deterministic cron execution via `JobSchedulerHostedService` | `StellaOps.Feedser.Core` tests | +| Trivy DB export | Deterministic digests across runs (ongoing TODO for end-to-end test) | `Exporter.TrivyDb` backlog | + +Budgets are enforced in automated tests where available; outstanding TODO/DOING +items (see task boards) continue tracking gaps such as exporter determinism. + +--- + +## 9 Testing + +* Unit and integration tests live alongside each component (`*.Tests`). +* Shared fixtures come from `StellaOps.Feedser.Testing` and + `StellaOps.Feedser.Tests.Shared` (linked via `Directory.Build.props`). +* Integration suites use ephemeral MongoDB and Redis via Testcontainers to + validate end-to-end flow without external dependencies. + +--- diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md new file mode 100755 index 00000000..7f456d02 --- /dev/null +++ b/docs/09_API_CLI_REFERENCE.md @@ -0,0 +1,329 @@ +# API & CLI Reference + +*Purpose* – give operators and integrators a single, authoritative spec for REST/GRPC calls **and** first‑party CLI tools (`stella-cli`, `zastava`, `stella`). +Everything here is *source‑of‑truth* for generated Swagger/OpenAPI and the `--help` screens in the CLIs. + +--- + +## 0 Quick Glance + +| Area | Call / Flag | Notes | +| ------------------ | ------------------------------------------- | ------------------------------------------------------------------------------ | +| Scan entry | `POST /scan` | Accepts SBOM or image; sub‑5 s target | +| Delta check | `POST /layers/missing` | <20 ms reply; powers *delta SBOM* feature | +| Rate‑limit / quota | — | Headers **`X‑Stella‑Quota‑Remaining`**, **`X‑Stella‑Reset`** on every response | +| Policy I/O | `GET /policy/export`, `POST /policy/import` | YAML now; Rego coming | +| Policy lint | `POST /policy/validate` | Returns 200 OK if ruleset passes | +| Auth | `POST /connect/token` (OpenIddict) | Client‑credentials preferred | +| Health | `GET /healthz` | Simple liveness probe | +| Attestation * | `POST /attest` (TODO Q1‑2026) | SLSA provenance + Rekor log | +| CLI flags | `--sbom-type` `--delta` `--policy-file` | Added to `stella` | + +\* Marked **TODO** → delivered after sixth month (kept on Feature Matrix “To Do” list). + +--- + +## 1 Authentication + +Stella Ops uses **OAuth 2.0 / OIDC** (token endpoint mounted via OpenIddict). + +``` +POST /connect/token +Content‑Type: application/x-www-form-urlencoded + +grant_type=client_credentials& +client_id=ci‑bot& +client_secret=REDACTED& +scope=stella.api +``` + +Successful response: + +```json +{ + "access_token": "eyJraWQi...", + "token_type": "Bearer", + "expires_in": 3600 +} +``` + +> **Tip** – pass the token via `Authorization: Bearer ` on every call. + +--- + +## 2 REST API + +### 2.0 Obtain / Refresh Offline‑Token + +```text +POST /token/offline +Authorization: Bearer +``` + +| Body field | Required | Example | Notes | +|------------|----------|---------|-------| +| `expiresDays` | no | `30` | Max 90 days | + +```json +{ + "jwt": "eyJhbGciOiJSUzI1NiIsInR5cCI6...", + "expires": "2025‑08‑17T00:00:00Z" +} +``` + +Token is signed with the backend’s private key and already contains +`"maxScansPerDay": {{ quota_token }}`. + + +### 2.1 Scan – Upload SBOM **or** Image + +``` +POST /scan +``` + +| Param / Header | In | Required | Description | +| -------------------- | ------ | -------- | --------------------------------------------------------------------- | +| `X‑Stella‑Sbom‑Type` | header | no | `trivy-json-v2`, `spdx-json`, `cyclonedx-json`; omitted ➞ auto‑detect | +| `?threshold` | query | no | `low`, `medium`, `high`, `critical`; default **critical** | +| body | body | yes | *Either* SBOM JSON *or* Docker image tarball/upload URL | + +Every successful `/scan` response now includes: + +| Header | Example | +|--------|---------| +| `X‑Stella‑Quota‑Remaining` | `129` | +| `X‑Stella‑Reset` | `2025‑07‑18T23:59:59Z` | +| `X‑Stella‑Token‑Expires` | `2025‑08‑17T00:00:00Z` | + +**Response 200** (scan completed): + +```json +{ + "digest": "sha256:…", + "summary": { + "Critical": 0, + "High": 3, + "Medium": 12, + "Low": 41 + }, + "policyStatus": "pass", + "quota": { + "remaining": 131, + "reset": "2025-07-18T00:00:00Z" + } +} +``` + +**Response 202** – queued; polling URL in `Location` header. + +--- + +### 2.2 Delta SBOM – Layer Cache Check + +``` +POST /layers/missing +Content‑Type: application/json +Authorization: Bearer +``` + +```json +{ + "layers": [ + "sha256:d38b...", + "sha256:af45..." + ] +} +``` + +**Response 200** — <20 ms target: + +```json +{ + "missing": [ + "sha256:af45..." + ] +} +``` + +Client then generates SBOM **only** for the `missing` layers and re‑posts `/scan`. + +--- + +### 2.3 Policy Endpoints + +| Method | Path | Purpose | +| ------ | ------------------ | ------------------------------------ | +| `GET` | `/policy/export` | Download live YAML ruleset | +| `POST` | `/policy/import` | Upload YAML or Rego; replaces active | +| `POST` | `/policy/validate` | Lint only; returns 400 on error | +| `GET` | `/policy/history` | Paginated change log (audit trail) | + +```yaml +# Example import payload (YAML) +version: "1.0" +rules: + - name: Ignore Low dev + severity: [Low, None] + environments: [dev, staging] + action: ignore +``` + +Validation errors come back as: + +```json +{ + "errors": [ + { + "path": "$.rules[0].severity", + "msg": "Invalid level 'None'" + } + ] +} +``` + +--- + +### 2.4 Attestation (Planned – Q1‑2026) + +``` +POST /attest +``` + +| Param | Purpose | +| ----------- | ------------------------------------- | +| body (JSON) | SLSA v1.0 provenance doc | +| | Signed + stored in local Rekor mirror | + +Returns `202 Accepted` and `Location: /attest/{id}` for async verify. + +--- + +### 2.5 Misc Endpoints + +| Path | Method | Description | +| ---------- | ------ | ---------------------------- | +| `/healthz` | GET | Liveness; returns `"ok"` | +| `/metrics` | GET | Prometheus exposition (OTel) | +| `/version` | GET | Git SHA + build date | + +--- + +## 3 First‑Party CLI Tools + +### 3.1 `stella` + +> *Package SBOM + Scan + Exit code* – designed for CI. + +``` +Usage: stella [OPTIONS] IMAGE_OR_SBOM +``` + +| Flag / Option | Default | Description | +| --------------- | ----------------------- | -------------------------------------------------- | +| `--server` | `http://localhost:8080` | API root | +| `--token` | *env `STELLA_TOKEN`* | Bearer token | +| `--sbom-type` | *auto* | Force `trivy-json-v2`/`spdx-json`/`cyclonedx-json` | +| `--delta` | `false` | Enable delta layer optimisation | +| `--policy-file` | *none* | Override server rules with local YAML/Rego | +| `--threshold` | `critical` | Fail build if ≥ level found | +| `--output-json` | *none* | Write raw scan result to file | +| `--wait-quota` | `true` | If 429 received, automatically wait `Retry‑After` and retry once. | + +**Exit codes** + +| Code | Meaning | +| ---- | ------------------------------------------- | +| 0 | Scan OK, policy passed | +| 1 | Vulnerabilities ≥ threshold OR policy block | +| 2 | Internal error (network etc.) | + +--- + +### 3.2 `stella‑zastava` + +> *Daemon / K8s DaemonSet* – watch container runtime, push SBOMs. + +Core flags (excerpt): + +| Flag | Purpose | +| ---------------- | ---------------------------------- | +| `--mode` | `listen` (default) / `enforce` | +| `--filter-image` | Regex; ignore infra/busybox images | +| `--threads` | Worker pool size | + +--- + +### 3.3 `stellopsctl` + +> *Admin utility* – policy snapshots, feed status, user CRUD. + +Examples: + +``` +stellopsctl policy export > policies/backup-2025-07-14.yaml +stellopsctl feed refresh # force OSV merge +stellopsctl user add dev-team --role developer +``` + +--- + +## 4 Error Model + +Uniform problem‑details object (RFC 7807): + +```json +{ + "type": "https://stella-ops.org/probs/validation", + "title": "Invalid request", + "status": 400, + "detail": "Layer digest malformed", + "traceId": "00-7c39..." +} +``` + +--- + +## 5 Rate Limits + +Default **40 requests / second / token**. +429 responses include `Retry-After` seconds header. + +--- + +## 6 FAQ & Tips + +* **Skip SBOM generation in CI** – supply a *pre‑built* SBOM and add `?sbom-only=true` to `/scan` for <1 s path. +* **Air‑gapped?** – point `--server` to `http://oukgw:8080` inside the Offline Update Kit. +* **YAML vs Rego** – YAML simpler; Rego unlocks time‑based logic (see samples). +* **Cosign verify plug‑ins** – enable `SCANNER_VERIFY_SIG=true` env to refuse unsigned plug‑ins. + +--- + +## 7 Planned Changes (Beyond 6 Months) + +These stay in *Feature Matrix → To Do* until design is frozen. + +| Epic / Feature | API Impact Sketch | +| ---------------------------- | ---------------------------------- | +| **SLSA L1‑L3** attestation | `/attest` (see §2.4) | +| Rekor transparency log | `/rekor/log/{id}` (GET) | +| Plug‑in Marketplace metadata | `/plugins/market` (catalog) | +| Horizontal scaling controls | `POST /cluster/node` (add/remove) | +| Windows agent support | Update LSAPI to PDE, no API change | + +--- + +## 8 References + +* OpenAPI YAML → `/openapi/v1.yaml` (served by backend) +* OAuth2 spec: +* SLSA spec: + +--- + +## 9 Changelog (truncated) + +* **2025‑07‑14** – added *delta SBOM*, policy import/export, CLI `--sbom-type`. +* **2025‑07‑12** – initial public reference. + +--- diff --git a/docs/10_OFFLINE_KIT.md b/docs/10_OFFLINE_KIT.md new file mode 100755 index 00000000..4a9c9d9c --- /dev/null +++ b/docs/10_OFFLINE_KIT.md @@ -0,0 +1,139 @@ +# Offline Update Kit (OUK) — 100 % Air‑Gap Operation + +> **Status:** ships together with the public α `v0.1.0` (ETA **late 2025**). +> All commands below assume the bundle name +> `stella-ouk‑2025‑α.tar.gz` – adjust once the real date tag is known. + +--- + +## 1 · What’s in the bundle 📦 + +| Item | Purpose | +|------|---------| +| **Vulnerability database** | Pre‑merged snapshot of NVD 2.0, OSV, GHSA
+ optional **regional catalogue** feeds | +| **Container images** | Scanner + Zastava for **x86‑64** & **arm64** | +| **Cosign signatures** | Release attestation & SBOM integrity | +| **SPDX SBOM** | Cryptographically signed bill of materials | +| **Import manifest** | Check‑sums & version metadata | + +Nightly **delta patches** keep the bundle < 350 MB while staying *T‑1 day* +current. + +--- + +## 2 · Download & verify 🔒 + +```bash +curl -LO https://get.stella-ops.org/releases/latest/stella-ops-offline-usage-kit-v0.1a.tar.gz +curl -LO https://get.stella-ops.org/releases/latest/stella-ops-offline-usage-kit-v0.1a.tar.gz.sig + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-usage-kit-v0.1a.tar.gz.sig \ + stella-ops-offline-usage-kit-v0.1a.tar.gz +``` + +The output shows `Verified OK` and the SHA‑256 digest ‑ compare with the +release notes. + +--- + +## 3 · Import on the isolated host 🚀 + +```bash +docker compose --env-file .env -f compose-stella.yml \ + exec stella-ops stella ouk import stella-ops-offline-usage-kit-v0.1a.tar.gz +``` + +* The scanner verifies the Cosign signature **before** activation. +* DB switch is atomic – **no downtime** for running jobs. +* Import time on an SSD VM ≈ 5‑7 s. + +--- + +## 4 · How the quota works offline 🔢 + +| Mode | Daily scans | Behaviour at 200 scans | Behaviour over limit | +| --------------- | ----------- | ---------------------- | ------------------------------------ | +| **Anonymous** | {{ quota_anon }} | Reminder banner | CLI slows \~10 % | +| **Token (JWT)** | {{ quota_token }} | Reminder banner | Throttle continues, **never blocks** | + +*Request a free JWT:* send a blank e‑mail to +`token@stella-ops.org` – the bot replies with a signed token that you +store as `STELLA_JWT` in **`.env`**. + +--- + +## 5 · Updating the bundle ⤴️ + +1. Download the newer tarball & signature. +2. Repeat the **verify‑blob** step. +3. Run `stella ouk import ` – only the delta applies; average + upgrade time is **< 3 s**. + +--- + +## 6 · Road‑map highlights for Sovereign 🌐 + +| Release | Planned feature | +| ---------------------- | ---------------------------------------- | +| **v0.1 α (late 2025)** | Manual OUK import • Zastava beta | +| **v0.3 β (Q2 2026)** | Auto‑apply delta patch • nightly re‑scan | +| **v0.4 RC (Q3 2026)** | LDAP/AD SSO • registry scanner GA | +| **v1.0 GA (Q4 2026)** | Custom TLS/crypto adaptors (**incl. SM2**)—enabled where law or security requires it | + +Full details live in the public [Road‑map](../roadmap/README.md). + +--- + +## 7 · Troubleshooting 🩹 + +| Symptom | Fix | +| -------------------------------------------- | ------------------------------------------------------- | +| `cosign: signature mismatch` | File corrupted ‑ re‑download both tarball & `.sig` | +| `ouk import: no space left` | Ensure **8 GiB** free in `/var/lib/docker` | +| Import succeeds but scans still hit Internet | Confirm `STELLA_AIRGAP=true` in `.env` (v0.1‑α setting) | + +--- + +## 8 · FAQ — abbreviated ❓ + +
+Does the JWT token work offline? + +Yes. Signature validation happens locally; no outbound call is made. + +
+ +
+Can I mirror the bundle internally? + +Absolutely. Host the tarball on an intranet HTTP/S server or an object +store; signatures remain valid. + +
+ +
+Is there a torrent alternative? + +Planned for the β releases – follow the +[community chat](https://matrix.to/#/#stellaops:libera.chat) for ETA. + +
+ +--- + +### Licence & provenance 📜 + +The Offline Update Kit is part of Stella Ops and therefore +**AGPL‑3.0‑or‑later**. All components inherit the same licence. + +```bash +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-usage-kit-v0.1a.tar.gz.sig \ + stella-ops-offline-usage-kit-v0.1a.tar.gz +``` + +— **Happy air‑gap scanning!** +© 2025‑2026 Stella Ops diff --git a/docs/10_PLUGIN_SDK_GUIDE.md b/docs/10_PLUGIN_SDK_GUIDE.md new file mode 100755 index 00000000..0956c046 --- /dev/null +++ b/docs/10_PLUGIN_SDK_GUIDE.md @@ -0,0 +1,194 @@ +# 10 · Plug‑in SDK Guide — **Stella Ops** +*(v 1.5 — 11 Jul 2025 · template install, no reload, IoC)* + +--- + +## 0 Audience & Scope +Guidance for developers who extend Stella Ops with schedule jobs, scanner adapters, TLS providers, notification channels, etc. Everything here is OSS; commercial variants simply ship additional signed plug‑ins. + +--- + +## 1 Prerequisites + +| Tool | Min Version | +| ----------------------- | ----------------------------------------------------------------- | +| .NET SDK | {{ dotnet }} | +| **StellaOps templates** | install once via `bash dotnet new install StellaOps.Templates::*` | +| **Cosign** | 2.3 + — used to sign DLLs | +| xUnit | 2.6 | +| Docker CLI | only if your plug‑in shells out to containers | + +--- + +## 2 Repository & Build Output + +Every plug‑in is hosted in **`git.stella‑ops.org`**. +At publish time it must copy its signed artefacts to: + +~~~text +src/backend/Stella.Ops.Plugin.Binaries// + ├── MyPlugin.dll + └── MyPlugin.dll.sig +~~~ + +The back‑end scans this folder on start‑up, verifies the **Cosign** signature, confirms the `[StellaPluginVersion]` gate, then loads the DLL inside an **isolated AssemblyLoadContext** to avoid dependency clashes + +--- + +## 3 Project Scaffold + +Generate with the installed template: + +~~~bash +dotnet new stellaops-plugin-schedule \ + -n MyPlugin.Schedule \ + --output src +~~~ + +Result: + +~~~text +src/ + ├─ MyPlugin.Schedule/ + │ ├─ MyJob.cs + │ └─ MyPlugin.Schedule.csproj + └─ tests/ + └─ MyPlugin.Schedule.Tests/ +~~~ + +--- + +## 4 MSBuild Wiring + +Add this to **`MyPlugin.Schedule.csproj`** so the signed DLL + `.sig` land in the canonical plug‑in folder: + +~~~xml + + $(SolutionDir)src/backend/Stella.Ops.Plugin.Binaries/$(MSBuildProjectName) + + + + + + + + + + + +~~~ + +--- + +## 5 Dependency‑Injection Entry‑point + +Back‑end auto‑discovers the static method below: + +~~~csharp +namespace StellaOps.DependencyInjection; + +public static class IoCConfigurator +{ + public static IServiceCollection Configure(this IServiceCollection services, + IConfiguration cfg) + { + services.AddSingleton(); // schedule job + services.Configure(cfg.GetSection("Plugins:MyPlugin")); + return services; + } +} +~~~ + +--- + +## 6 Schedule Plug‑ins + +### 6.1 Minimal Job + +~~~csharp +using StellaOps.Scheduling; // contract + +[StellaPluginVersion("2.0.0")] +public sealed class MyJob : IJob +{ + public async Task ExecuteAsync(CancellationToken ct) + { + Console.WriteLine("Hello from plug‑in!"); + await Task.Delay(500, ct); + } +} +~~~ + +### 6.2 Cron Registration + +```csharp +services.AddCronJob("0 15 * * *"); // everyday +``` + +15:00 +Cron syntax follows Hangfire rules  + +## 7 Scanner Adapters + +Implement IScannerRunner. +Register inside Configure: +```csharp +services.AddScanner("alt"); // backend +``` + +selects by --engine alt +If the engine needs a side‑car container, include a Dockerfile in your repo and document resource expectations. +## 8 Packaging & Signing + +```bash +dotnet publish -c Release -p:PublishSingleFile=true -o out +cosign sign --key $COSIGN_KEY out/MyPlugin.Schedule.dll # sign binary only +sha256sum out/MyPlugin.Schedule.dll > out/.sha256 # optional checksum +zip MyPlugin.zip out/* README.md +``` + +Unsigned DLLs are refused when StellaOps:Security:DisableUnsigned=false. + +## 9 Deployment + +```bash +docker cp MyPlugin.zip :/opt/plugins/ && docker restart +``` + +Check /health – "plugins":["MyPlugin.Schedule@2.0.0"]. +(Hot‑reload was removed to keep the core process simple and memory‑safe.) + +## 10 Configuration Patterns + +| Need | Pattern | +| ------------ | --------------------------------------------------------- | +| Settings | Plugins:MyPlugin:* in appsettings.json. | +| Secrets | Redis secure:: (encrypted per TLS provider). | +| Dynamic cron | Implement ICronConfigurable; UI exposes editor. | + +## 11 Testing & CI + +| Layer | Tool | Gate | +| ----------- | -------------------------- | ------------------- | +| Unit | xUnit + Moq | ≥ 50 % lines | +| Integration | Testcontainers ‑ run in CI | Job completes < 5 s | +| Style | dotnet | format 0 warnings | + +Use the pre‑baked workflow in StellaOps.Templates as starting point. + +## 12 Publishing to the Community Marketplace + +Tag Git release plugin‑vX.Y.Z and attach the signed ZIP. +Submit a PR to stellaops/community-plugins.json with metadata & git URL. +On merge, the plug‑in shows up in the UI Marketplace. + +## 13 Common Pitfalls + +| Symptom | Root cause | Fix | +| ------------------- | -------------------------- | ------------------------------------------- | +| NotDetected | .sig missing | cosign sign … | +| VersionGateMismatch | Backend 2.1 vs plug‑in 2.0 | Re‑compile / bump attribute | +| FileLoadException | Duplicate | StellaOps.Common Ensure PrivateAssets="all" | +| Redis | timeouts Large writes | Batch or use Mongo | \ No newline at end of file diff --git a/docs/11_DATA_SCHEMAS.md b/docs/11_DATA_SCHEMAS.md new file mode 100755 index 00000000..b68ac7ec --- /dev/null +++ b/docs/11_DATA_SCHEMAS.md @@ -0,0 +1,196 @@ +# Data Schemas & Persistence Contracts + +*Audience* – backend developers, plug‑in authors, DB admins. +*Scope* – describes **Redis**, **MongoDB** (optional), and on‑disk blob shapes that power Stella Ops. + +--- + +## 0 Document Conventions + +* **CamelCase** for JSON. +* All timestamps are **RFC 3339 / ISO 8601** with `Z` (UTC). +* `⭑` = planned but *not* shipped yet (kept on Feature Matrix “To Do”). + +--- + +## 1 SBOM Wrapper Envelope + +Every SBOM blob (regardless of format) is stored on disk or in object storage with a *sidecar* JSON file that indexes it for the scanners. + +#### 1.1 JSON Shape + +```jsonc +{ + "id": "sha256:417f…", // digest of the SBOM *file* itself + "imageDigest": "sha256:e2b9…", // digest of the original container image + "created": "2025-07-14T07:02:13Z", + "format": "trivy-json-v2", // NEW enum: trivy-json-v2 | spdx-json | cyclonedx-json + "layers": [ + "sha256:d38b…", // layer digests (ordered) + "sha256:af45…" + ], + "partial": false, // true => delta SBOM (only some layers) + "provenanceId": "prov_0291" // ⭑ link to SLSA attestation (Q1‑2026) +} +``` + +*`format`* **NEW** – added to support **multiple SBOM formats**. +*`partial`* **NEW** – true when generated via the **delta SBOM** flow (§1.3). + +#### 1.2 File‑system Layout + +``` +blobs/ + ├─ 417f… # digest prefix + │   ├─ sbom.json # payload (any format) + │   └─ sbom.meta.json # wrapper (shape above) +``` + +> **Note** – blob storage can point at S3, MinIO, or plain disk; driver plug‑ins adapt. + +#### 1.3 Delta SBOM Extension + +When `partial: true`, *only* the missing layers have been scanned. +Merging logic inside `scanning` module stitches new data onto the cached full SBOM in Redis. + +--- + +## 2 Redis Keyspace + +| Key pattern | Type | TTL | Purpose | +|-------------------------------------|---------|------|--------------------------------------------------| +| `scan:<digest>` | string | ∞ | Last scan JSON result (as returned by `/scan`) | +| `layers:<digest>` | set | 90d | Layers already possessing SBOMs (delta cache) | +| `policy:active` | string | ∞ | YAML **or** Rego ruleset | +| `quota:<token>` | string | *until next UTC midnight* | Per‑token scan counter for Free tier ({{ quota_token }} scans). | +| `policy:history` | list | ∞ | Change audit IDs (see Mongo) | +| `feed:nvd:json` | string | 24h | Normalised feed snapshot | +| `locator:<imageDigest>` | string | 30d | Maps image digest → sbomBlobId | +| `metrics:…` | various | — | Prom / OTLP runtime metrics | + +> **Delta SBOM** uses `layers:*` to skip work in <20 ms. +> **Quota enforcement** increments `quota:` atomically; when {{ quota_token }} the API returns **429**. + +--- + +## 3 MongoDB Collections (Optional) + +Only enabled when `MONGO_URI` is supplied (for long‑term audit). + +| Collection | Shape (summary) | Indexes | +|--------------------|------------------------------------------------------------|-------------------------------------| +| `sbom_history` | Wrapper JSON + `replaceTs` on overwrite | `{imageDigest}` `{created}` | +| `policy_versions` | `{_id, yaml, rego, authorId, created}` | `{created}` | +| `attestations` ⭑ | SLSA provenance doc + Rekor log pointer | `{imageDigest}` | +| `audit_log` | Fully rendered RFC 5424 entries (UI & CLI actions) | `{userId}` `{ts}` | + +Schema detail for **policy_versions**: + +```jsonc +{ + "_id": "6619e90b8c5e1f76", + "yaml": "version: 1.0\nrules:\n - …", + "rego": null, // filled when Rego uploaded + "authorId": "u_1021", + "created": "2025-07-14T08:15:04Z", + "comment": "Imported via API" +} +``` + +--- + +## 4 Policy Schema (YAML v1.0) + +Minimal viable grammar (subset of OSV‑SCHEMA ideas). + +```yaml +version: "1.0" +rules: + - name: Block Critical + severity: [Critical] + action: block + - name: Ignore Low Dev + severity: [Low, None] + environments: [dev, staging] + action: ignore + expires: "2026-01-01" + - name: Escalate RegionalFeed High + sources: [NVD, CNNVD, CNVD, ENISA, JVN, BDU] + severity: [High, Critical] + action: escalate +``` + +Validation is performed by `policy:mapping.yaml` JSON‑Schema embedded in backend. + +### 4.1 Rego Variant (Advanced – TODO) + +*Accepted but stored as‑is in `rego` field.* +Evaluated via internal **OPA** side‑car once feature graduates from TODO list. + +--- + +## 5 SLSA Attestation Schema ⭑ + +Planned for Q1‑2026 (kept here for early plug‑in authors). + +```jsonc +{ + "id": "prov_0291", + "imageDigest": "sha256:e2b9…", + "buildType": "https://slsa.dev/container/v1", + "builder": { + "id": "https://git.stella-ops.ru/ci/stella-runner@sha256:f7b7…" + }, + "metadata": { + "invocation": { + "parameters": {"GIT_SHA": "f6a1…"}, + "buildStart": "2025-07-14T06:59:17Z", + "buildEnd": "2025-07-14T07:01:22Z" + }, + "completeness": {"parameters": true} + }, + "materials": [ + {"uri": "git+https://git…", "digest": {"sha1": "f6a1…"}} + ], + "rekorLogIndex": 99817 // entry in local Rekor mirror +} +``` + +--- + +## 6 Validator Contracts + +* For SBOM wrapper – `ISbomValidator` (DLL plug‑in) must return *typed* error list. +* For YAML policies – JSON‑Schema at `/schemas/policy‑v1.json`. +* For Rego – OPA `opa eval --fail-defined` under the hood. +* For **Free‑tier quotas** – `IQuotaService` integration tests ensure `quota:` resets at UTC midnight and produces correct `Retry‑After` headers. + +--- + +## 7 Migration Notes + +1. **Add `format` column** to existing SBOM wrappers; default to `trivy-json-v2`. +2. **Populate `layers` & `partial`** via backfill script (ship with `stellopsctl migrate` wizard). +3. Policy YAML previously stored in Redis → copy to Mongo if persistence enabled. +4. Prepare `attestations` collection (empty) – safe to create in advance. + +--- + +## 8 Open Questions / Future Work + +* How to de‑duplicate *identical* Rego policies differing only in whitespace? +* Embed *GOST 34.11‑2018* digests when users enable Russian crypto suite? +* Should enterprise tiers share the same Redis quota keys or switch to JWT claim `tier != Free` bypass? +* Evaluate sliding‑window quota instead of strict daily reset. +* Consider rate‑limit for `/layers/missing` to avoid brute‑force enumeration. + +--- + +## 9 Change Log + +| Date | Note | +|------------|--------------------------------------------------------------------------------| +| 2025‑07‑14 | **Added:** `format`, `partial`, delta cache keys, YAML policy schema v1.0. | +| 2025‑07‑12 | **Initial public draft** – SBOM wrapper, Redis keyspace, audit collections. | + +--- diff --git a/docs/11_GOVERNANCE.md b/docs/11_GOVERNANCE.md new file mode 100755 index 00000000..2b835033 --- /dev/null +++ b/docs/11_GOVERNANCE.md @@ -0,0 +1,93 @@ +# Stella Ops Project Governance +*Lazy Consensus • Maintainer Charter • Transparent Veto* + +> **Scope** – applies to **all** repositories under +> `https://git.stella-ops.org/stella-ops/*` unless a sub‑project overrides it +> with its own charter approved by the Core Maintainers. + +--- + +## 1 · Decision‑making workflow 🗳️ + +| Stage | Default vote | Timer | +|-------|--------------|-------| +| **Docs / non‑code PR** | `+1` | **48 h** | +| **Code / tests PR** | `+1` | **7 × 24 h** | +| **Security‑sensitive / breaking API** | `+1` + explicit **`security‑LGTM`** | **7 × 24 h** | + +**Lazy‑consensus** – silence = approval once the timer elapses. + +* **Veto `‑1`** must include a concrete concern **and** a path to resolution. +* After 3 unresolved vetoes the PR escalates to a **Maintainer Summit** call. + +--- + +## 2 · Maintainer approval thresholds 👥 + +| Change class | Approvals required | Example | +|--------------|-------------------|---------| +| **Trivial** | 0 | Typos, comment fixes | +| **Non‑trivial** | **2 Maintainers** | New API endpoint, feature flag | +| **Security / breaking** | Lazy‑consensus **+ `security‑LGTM`** | JWT validation, crypto swap | + +Approval is recorded via Git forge review or a signed commit trailer +`Signed-off-by: `. + +--- + +## 3 · Becoming (and staying) a Maintainer 🌱 + +1. **3 + months** of consistent, high‑quality contributions. +2. **Nomination** by an existing Maintainer via issue. +3. **7‑day vote** – needs ≥ **⅔ majority** “`+1`”. +4. Sign `MAINTAINER_AGREEMENT.md` and enable **2FA**. +5. Inactivity > 6 months → automatic emeritus status (can be re‑activated). + +--- + +## 4 · Release authority & provenance 🔏 + +* Every tag is **co‑signed by at least one Security Maintainer**. +* CI emits a **signed SPDX SBOM** + **Cosign provenance**. +* Release cadence is fixed – see [public Road‑map](../roadmap/README.md). +* Security fixes may create out‑of‑band `x.y.z‑hotfix` tags. + +--- + +## 5 · Escalation lanes 🚦 + +| Situation | Escalation | +|-----------|------------| +| Technical deadlock | **Maintainer Summit** (recorded & published) | +| Security bug | Follow [Security Policy](../security/01_SECURITY_POLICY.md) | +| Code of Conduct violation | See `12_CODE_OF_CONDUCT.md` escalation ladder | + +--- + +## 6 · Contribution etiquette 🤝 + +* Draft PRs early – CI linting & tests help you iterate. +* “There are no stupid questions” – ask in **Matrix #dev**. +* Keep commit messages in **imperative mood** (`Fix typo`, `Add SBOM cache`). +* Run the `pre‑commit` hook locally before pushing. + +--- + +## 7 · Licence reminder 📜 + +Stella Ops is **AGPL‑3.0‑or‑later**. By contributing you agree that your +patches are released under the same licence. + +--- + +### Appendix A – Maintainer list 📇 + +*(Generated via `scripts/gen-maintainers.sh` – edit the YAML, **not** this +section directly.)* + +| Handle | Area | Since | +|--------|------|-------| +| `@alice` | Core scanner • Security | 2025‑04 | +| `@bob` | UI • Docs | 2025‑06 | + +--- \ No newline at end of file diff --git a/docs/12_CODE_OF_CONDUCT.md b/docs/12_CODE_OF_CONDUCT.md new file mode 100755 index 00000000..236b4ab8 --- /dev/null +++ b/docs/12_CODE_OF_CONDUCT.md @@ -0,0 +1,88 @@ +# Stella Ops Code of Conduct +*Contributor Covenant v2.1 + project‑specific escalation paths* + +> We pledge to make participation in the Stella Ops community a +> harassment‑free experience for everyone, regardless of age, body size, +> disability, ethnicity, sex characteristics, gender identity and expression, +> level of experience, education, socio‑economic status, nationality, +> personal appearance, race, religion, or sexual identity and orientation. + +--- + +## 0 · Our standard + +This project adopts the +[**Contributor Covenant v2.1**](https://www.contributor-covenant.org/version/2/1/code_of_conduct/) +with the additions and clarifications listed below. +If anything here conflicts with the upstream covenant, *our additions win*. + +--- + +## 1 · Scope + +| Applies to | Examples | +|------------|----------| +| **All official spaces** | Repos under `git.stella-ops.org/stella-ops.org/*`, Matrix rooms (`#stellaops:*`), issue trackers, pull‑request reviews, community calls, and any event officially sponsored by Stella Ops | +| **Unofficial spaces that impact the project** | Public social‑media posts that target or harass community members, coordinated harassment campaigns, doxxing, etc. | + +--- + +## 2 · Reporting a violation ☎️ + +| Channel | When to use | +|---------|-------------| +| `conduct@stella-ops.org` (PGP key [`keys/#pgp`](../keys/#pgp)) | **Primary, confidential** – anything from micro‑aggressions to serious harassment | +| Matrix `/msg @coc-bot:libera.chat` | Quick, in‑chat nudge for minor issues | +| Public issue with label `coc` | Transparency preferred and **you feel safe** doing so | + +We aim to acknowledge **within 48 hours** (business days, UTC). + +--- + +## 3 · Incident handlers 🛡️ + +| Name | Role | Alt‑contact | +|------|------|-------------| +| Alice Doe (`@alice`) | Core Maintainer • Security WG | `+1‑555‑0123` | +| Bob Ng (`@bob`) | UI Maintainer • Community lead | `+1‑555‑0456` | + +If **any** handler is the subject of a complaint, skip them and contact another +handler directly or email `conduct@stella-ops.org` only. + +--- + +## 4 · Enforcement ladder ⚖️ + +1. **Private coaches / mediation** – first attempt to resolve misunderstandings. +2. **Warning** – written, includes corrective actions & cooling‑off period. +3. **Temporary exclusion** – mute (chat), read‑only (repo) for *N* days. +4. **Permanent ban** – removal from all official spaces + revocation of roles. + +All decisions are documented **privately** (for confidentiality) but a summary +is published quarterly in the “Community Health” report. + +--- + +## 5 · Appeals 🔄 + +A sanctioned individual may appeal **once** by emailing +`appeals@stella-ops.org` within **14 days** of the decision. +Appeals are reviewed by **three maintainers not involved in the original case** +and resolved within 30 days. + +--- + +## 6 · No‑retaliation policy 🛑 + +Retaliation against reporters **will not be tolerated** and results in +immediate progression to **Step 4** of the enforcement ladder. + +--- + +## 7 · Attribution & licence 📜 + +* Text adapted from Contributor Covenant v2.1 – + Copyright © 2014‑2024 Contributor Covenant Contributors + Licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). + +--- diff --git a/docs/12_PERFORMANCE_WORKBOOK.md b/docs/12_PERFORMANCE_WORKBOOK.md new file mode 100755 index 00000000..192fecc3 --- /dev/null +++ b/docs/12_PERFORMANCE_WORKBOOK.md @@ -0,0 +1,167 @@ +# 12 - Performance Workbook + +*Purpose* – define **repeatable, data‑driven** benchmarks that guard Stella Ops’ core pledge: +> *“P95 vulnerability feedback in ≤ 5 seconds.”* + +--- + +## 0 Benchmark Scope + +| Area | Included | Excluded | +|------------------|----------------------------------|---------------------------| +| SBOM‑first scan | Trivy engine w/ warmed DB | Full image unpack ≥ 300 MB | +| Delta SBOM ⭑ | Missing‑layer lookup & merge | Multi‑arch images | +| Policy eval ⭑ | YAML → JSON → rule match | Rego (until GA) | +| Feed merge | NVD JSON 2023–2025 | GHSA GraphQL (plugin) | +| Quota wait‑path | 5 s soft‑wait, 60 s hard‑wait behaviour | Paid tiers (unlimited) | +| API latency | REST `/scan`, `/layers/missing` | UI SPA calls | + +⭑ = new in July 2025. + +--- + +## 1 Hardware Baseline (Reference Rig) + +| Element | Spec | +|-------------|------------------------------------| +| CPU | 8 vCPU (Intel Ice‑Lake equiv.) | +| Memory | 16 GiB | +| Disk | NVMe SSD, 3 GB/s R/W | +| Network | 1 Gbit virt. switch | +| Container | Docker 25.0 + overlay2 | +| OS | Ubuntu 22.04 LTS (kernel 6.8) | + +*All P95 targets assume a **single‑node** deployment on this rig unless stated.* + +--- + +## 2 Phase Targets & Gates + +| Phase (ID) | Target P95 | Gate (CI) | Rationale | +|-----------------------|-----------:|-----------|----------------------------------------| +| **SBOM_FIRST** | ≤ 5 s | `hard` | Core UX promise. | +| **IMAGE_UNPACK** | ≤ 10 s | `soft` | Fallback path for legacy flows. | +| **DELTA_SBOM** ⭑ | ≤ 1 s | `hard` | Needed to stay sub‑5 s for big bases. | +| **POLICY_EVAL** ⭑ | ≤ 50 ms | `hard` | Keeps gate latency invisible to users. | +| **QUOTA_WAIT** ⭑ | *soft* ≤ 5 s
*hard* ≤ 60 s | `hard` | Ensures graceful Free‑tier throttling. | +| **SCHED_RESCAN** | ≤ 30 s | `soft` | Nightly batch – not user‑facing. | +| **FEED_MERGE** | ≤ 60 s | `soft` | Off‑peak cron @ 01:00. | +| **API_P95** | ≤ 200 ms | `hard` | UI snappiness. | + +*Gate* legend — `hard`: break CI if regression > 3 × target, +`soft`: raise warning & issue ticket. + +--- + +## 3 Test Harness + +* **Runner** – `perf/run.sh`, accepts `--phase` and `--samples`. +* **Metrics** – Prometheus + `jq` extracts; aggregated via `scripts/aggregate.ts`. +* **CI** – GitLab CI job *benchmark* publishes JSON to `bench‑artifacts/`. +* **Visualisation** – Grafana dashboard *Stella‑Perf* (provisioned JSON). + +> **Note** – harness mounts `/var/cache/trivy` tmpfs to avoid disk noise. + +--- + +## 4 Current Results (July 2025) + +| Phase | Samples | Mean (s) | P95 (s) | Target OK? | +|---------------|--------:|---------:|--------:|-----------:| +| SBOM_FIRST | 100 | 3.7 | 4.9 | ✅ | +| IMAGE_UNPACK | 50 | 6.4 | 9.2 | ✅ | +| **DELTA_SBOM**| 100 | 0.46 | 0.83 | ✅ | +| **POLICY_EVAL** | 1 000 | 0.021 | 0.041 | ✅ | +| **QUOTA_WAIT** | 80 | 4.0* | 4.9* | ✅ | +| SCHED_RESCAN | 10 | 18.3 | 24.9 | ✅ | +| FEED_MERGE | 3 | 38.1 | 41.0 | ✅ | +| API_P95 | 20 000 | 0.087 | 0.143 | ✅ | + +*Data files:* `bench-artifacts/2025‑07‑14/phase‑stats.json`. + +--- + +## 5 Δ‑SBOM Micro‑Benchmark Detail + +### 5.1 Scenario + +1. Base image `python:3.12-slim` already scanned (all layers cached). +2. Application layer (`COPY . /app`) triggers new digest. +3. `Stella CLI` lists **7** layers, backend replies *6 hit*, *1 miss*. +4. Builder scans **only 1 layer** (~9 MiB, 217 files) & uploads delta. + +### 5.2 Key Timings + +| Step | Time (ms) | +|---------------------|----------:| +| `/layers/missing` | 13 | +| Trivy single layer | 655 | +| Upload delta blob | 88 | +| Backend merge + CVE | 74 | +| **Total wall‑time** | **830 ms** | + +--- + +## 6 Quota Wait‑Path Benchmark Detail + +### 6.1 Scenario + +1. Free‑tier token reaches **scan #200** – dashboard shows yellow banner. + +### 6.2 Key Timings + +| Step | Time (ms) | +|------------------------------------|----------:| +| `/quota/check` Redis LUA INCR | 0.8 | +| Soft wait sleep (server) | 5 000 | +| Hard wait sleep (server) | 60 000 | +| End‑to‑end wall‑time (soft‑hit) | 5 003 | +| End‑to‑end wall‑time (hard‑hit) | 60 004 | + +--- +## 7 Policy Eval Bench + +### 7.1 Setup + +* Policy YAML: **28** rules, mix severity & package conditions. +* Input: scan result JSON with **1 026** findings. +* Evaluator: custom rules engine (Go structs → map look‑ups). + +### 7.2 Latency Histogram + +``` +0‑10 ms ▇▇▇▇▇▇▇▇▇▇ 38 % +10‑20 ms ▇▇▇▇▇▇▇▇▇▇ 42 % +20‑40 ms ▇▇▇▇▇▇ 17 % +40‑50 ms ▇ 3 % +``` + +P99 = 48 ms. Meets 50 ms gate. + +--- + +## 8 Trend Snapshot + +![Perf trend spark‑line placeholder](perf‑trend.svg) + +_Plot generated weekly by `scripts/update‑trend.py`; shows last 12 weeks P95 per phase._ + +--- + +## 9 Action Items + +1. **Image Unpack** – Evaluate zstd for layer decompress; aim to shave 1 s. +2. **Feed Merge** – Parallelise regional XML feed parse (plugin) once stable. +3. **Rego Support** – Prototype OPA side‑car; target ≤ 100 ms eval. +4. **Concurrency** – Stress‑test 100 rps on 4‑node Redis cluster (Q4‑2025). + +--- + +## 10 Change Log + +| Date | Note | +|------------|-------------------------------------------------------------------------| +| 2025‑07‑14 | Added Δ‑SBOM & Policy Eval phases; updated targets & current results. | +| 2025‑07‑12 | First public workbook (SBOM‑first, image‑unpack, feed merge). | + +--- diff --git a/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md b/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md new file mode 100755 index 00000000..2586f655 --- /dev/null +++ b/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md @@ -0,0 +1,209 @@ +# 13 · Release Engineering Playbook — Stella Ops + + +A concise, automation‑first guide describing **how source code on `main` becomes a verifiably signed, air‑gap‑friendly release**. +It is opinionated for offline use‑cases and supply‑chain security (SLSA ≥ level 2 today, aiming for level 3). + +--- + +## 0 Release Philosophy + +* **Fast but fearless** – every commit on `main` must be releasable; broken builds break the build, not the team. +* **Reproducible** – anyone can rebuild byte‑identical artefacts with a single `make release` offline. +* **Secure by default** – every artefact ships with a SBOM, Cosign signature and (future) Rekor log entry. +* **Offline‑first** – all dependencies are vendored or mirrored into the internal registry; no Internet required at runtime. + +--- + +## 1 Versioning & Branching + +| Branch | Purpose | Auto‑publish? | +| ------------- | ------------------------------ | --------------------------------------- | +| `main` | Always‑green development trunk | `nightly-*` images | +| `release/X.Y` | Stabilise a minor line | `stella:X.Y-rcN` | +| Tags | `X.Y.Z` = SemVer | `stella:X.Y.Z`, OUK tarball, Helm chart | + +* **SemVer** – MAJOR for breaking API/CLI changes, MINOR for features, PATCH for fixes. +* Release tags are **signed** (`git tag -s`) with the Stella Ops GPG key (`0x90C4…`). + +--- + +## 2 CI/CD Overview (GitLab CI + GitLab Runner) + +```mermaid +graph LR + A[push / MR] --> Lint + Lint --> Unit + Unit --> Build + Build --> Test-Container + Test-Container --> SBOM + SBOM --> Sign + Sign --> Publish + Publish --> E2E + Publish --> Notify +``` + +### Pipeline Stages + +| Stage | Key tasks | +| ------------------ | ------------------------------------------------------------------------------------------------ | +| **Lint** | ESLint, golangci‑lint, hadolint, markdown‑lint. | +| **Unit** | `dotnet test`, `go test`, Jest UI tests. | +| **Quota unit‑tests 🏷** | Validate QuotaService logic: reset at UTC, 5 s vs 60 s waits, header correctness. | +| **Build** | Multi‑arch container build (`linux/amd64`, `linux/arm64`) using **BuildKit** + `--provenance` 📌. | +| **Test‑Container** | Spin up compose file, run smoke APIs. | +| **SBOM** 📌 | Invoke **StellaOps.SBOMBuilder** to generate SPDX JSON + attach `.sbom` label to image. | +| **Sign** | Sign image with **Cosign** (`cosign sign --key cosign.key`). | +| **Publish** | Push to `registry.git.stella-ops.org`. | +| **E2E** | Kind‑based Kubernetes test incl. Zastava DaemonSet; verify sub‑5 s scan SLA. | +| **Notify** | Report to Mattermost & GitLab Slack app. | +| **OfflineToken** | Call `JwtIssuer.Generate(exp=30d)` → store `client.jwt` artefact → attach to OUK build context | + +*All stages run in parallel where possible; max wall‑time < 15 min.* + +--- + +## 3 Container Image Strategy + +| Image | Registry Tag | Contents | +| ------------------------------ | --------------------------- | ---------------------------------------------------------------------- | +| **backend** | `stella/backend:{ver}` | ASP.NET API, plugin loader. | +| **ui** | `stella/ui:{ver}` | Pre‑built Angular SPA. | +| **runner-trivy** | `stella/runner-trivy:{ver}` | Trivy CLI + SPDX/CycloneDX 🛠. | +| **runner-grype** | `stella/runner-grype:{ver}` | Optional plug‑in scanner. | +| **🏷️ StellaOps.Registry** 📌 | `stella/registry:{ver}` | Scratch image embedding Docker Registry v2 + Cosign policy controller. | +| **🏷️ StellaOps.MutePolicies** 📌 | `stella/policies:{ver}` | Sidecar serving policy bundles. | +| **🏷️ StellaOps.Attestor** 📌 | `stella/attestor:{ver}` | SLSA provenance & Rekor signer (future). | + +*Images are **`--label org.opencontainers.image.source=git.stella-ops.ru`** and include SBOMs generated at build time.* + +--- + +## 4 📌 Offline Update Kit (OUK) Build & Distribution + +**Purpose** – deliver updated CVE feeds & Trivy DB to air‑gapped clusters. + +### 4.1 CLI Tool + +*Go binary `ouk` lives in `tools/ouk/`.* + +```sh +ouk fetch \ + --nvd --osv \ + --trivy-db --date $(date -I) \ + --output ouk-$(date +%Y%m%d).tar.gz \ + --sign cosign.key +``` + +### 4.2 Pipeline Hook + +* Runs on **first Friday** each month (cron). +* Generates tarball, signs it, uploads to **GitLab Release asset**. +* SHA‑256 + signature published alongside. + +### 4.3 Activation Flow (runtime) + +1. Admin uploads `.tar.gz` via **UI → Settings → Offline Updates (OUK)**. +2. Backend verifies Cosign signature & digest. +3. Files extracted into `var/lib/stella/db`. +4. Redis caches invalidated; Dashboard “Feed Age” ticks green. +5. Audit event `ouk_update` stored. + +### 4.4 Token Detail + +client.jwt placed under /root/ inside the tarball. +CI job fails if token expiry < 29 days (guard against stale caches). + +--- + +## 5 Artifact Signing & Transparency + +| Artefact | Signer | Tool | +| ------------ | --------------- | --------------------- | +| Git tags | GPG (`0x90C4…`) | `git tag -s` | +| Containers | Cosign key pair | `cosign sign` | +| Helm Charts | prov file | `helm package --sign` | +| OUK tarballs | Cosign | `cosign sign-blob` | + +**Rekor** integration is **TODO** – once the internal Rekor mirror is online (`StellaOpsAttestor`) a post‑publish job will submit transparency log entries. + +--- + +## 6 Release Checklist + +1. CI pipeline green. +2. Bump `VERSION` file. +3. Tag `git tag -s X.Y.Z -m "Release X.Y.Z"` & push. +4. GitLab CI auto‑publishes images & charts. +5. Draft GitLab **Release Notes** using `tools/release-notes-gen`. +6. Verify SBOM attachment with `stella sbom verify stella/backend:X.Y.Z`. +7. Smoke‑test OUK tarball in offline lab. +8. Announce in `#stella-release` Mattermost channel. + +--- + +## 7 Hot‑fix Procedure + +* Branch from latest tag → `hotfix/X.Y.Z+1-hf1`. +* Apply minimal patch, add regression test. +* CI pipeline (with reduced stages) must pass. +* Tag `X.Y.Z+1`. +* Publish only container + Helm chart; OUK not rebuilt. +* Cherry‑pick back to `main`. + +--- + +## 8 Deprecation & End‑of‑Life Policy + +| Feature | Deprecation notice | Removal earliest | +| ------------------------ | ------------------ | ---------------- | +| Legacy CSV policy import | 2025‑10‑01 | 2026‑04‑01 | +| Docker v1 Registry auth | 2025‑12‑01 | 2026‑06‑01 | +| In‑image Trivy DB | 2025‑12‑15 | 2026‑03‑15 | + +*At least 6 months notice; removal requires major version bump.* + +--- + +## 9 📌 Non‑Commercial Usage Rules (English canonical) + +1. **Free for internal security assessments** (company or personal). +2. **SaaS resale / re‑hosting prohibited** without prior written consent (AGPL §13). +3. If you distribute a fork with UI or backend modifications **you must**: + * Publish the complete modified source code. + * Retain the original Stella Ops attribution in UI footer and CLI `--version`. +4. All third‑party dependencies remain under their respective licences (MIT, Apache‑2.0, ISC, BSD). +5. Deployments in state‑regulated or classified environments must obey**applicable local regulations** governing cryptography and software distribution. + +--- + +## 10 Best Practices Snapshot 📌 + +* **SBOM‑per‑image** → attach at build time; store as OCI artifact for supply‑chain introspection. +* **Provenance flag** (`--provenance=true`) in BuildKit fulfils SLSA 2 requirement. +* Use **multi‑arch, reproducible builds** (`SOURCE_DATE_EPOCH` pins timestamps). +* All pipelines enforce **Signed‑off‑by (DCO)**; CI fails if trailer missing. +* `cosign policy` ensures only images signed by the project key run in production. + +--- + +## 11 Contributing to Release Engineering + +* Fork & create MR to `infra/release-*`. +* All infra changes require green **`integration-e2e-offline`** job. +* Discuss larger infra migrations in `#sig-release` Mattermost; decisions recorded in `ADR/` folder. + +--- + +## 12 Change Log (high‑level) + +| Version | Date | Note | +| ------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| v2.1 | 2025‑07‑15 | Added OUK build/publish pipeline, internal registry image (`StellaOps.Registry`), non‑commercial usage rules extraction, SBOM stage, BuildKit provenance. | +| v2.0 | 2025‑07‑12 | Initial open‑sourcing of Release Engineering guide. | +| v1.1 | 2025‑07‑09 | Fixed inner fencing; added retention policy | +| v1.0 | 2025‑07‑09 | Initial playbook | + +--- + +*(End of Release Engineering Playbook v1.1)* diff --git a/docs/13_SECURITY_POLICY.md b/docs/13_SECURITY_POLICY.md new file mode 100755 index 00000000..1956a68d --- /dev/null +++ b/docs/13_SECURITY_POLICY.md @@ -0,0 +1,101 @@ +# Stella Ops Security Policy & Responsible Disclosure +*Version 3 · 2025‑07‑15* + +--- + +## 0 · Supported versions 🗓️ + +| Release line | Status | Security fix window | +|--------------|--------|---------------------| +| **v0.1 α** (late 2025) | *Upcoming* | 90 days after GA of v0.2 | +| **v0.2 β** (Q1 2026) | *Planned* | 6 months after GA of v0.3 | +| **v0.3 β** (Q2 2026) | *Planned* | 6 months after GA of v0.4 | +| **v0.4 RC** (Q3 2026) | *Planned* | Until v1.0 GA | +| **v1.0 GA** (Q4 2026) | *Future LTS* | 24 months from release | + +Pre‑GA lines receive **critical** and **high**‑severity fixes only. + +--- + +## 1 · How to report a vulnerability 🔒 + +| Channel | PGP‑encrypted? | Target SLA | +|---------|---------------|-----------| +| `security@stella-ops.org` | **Yes** – PGP key: [`/keys/#pgp`](../keys/#pgp) | 72 h acknowledgement | +| Matrix DM → `@sec‑bot:libera.chat` | Optional | 72 h acknowledgement | +| Public issue with label `security` | No (for non‑confidential flaws) | 7 d acknowledgement | + +Please include: + +* Affected version(s) and environment +* Reproduction steps or PoC +* Impact assessment (data exposure, RCE, DoS, etc.) +* Preferred disclosure timeline / CVE request info + +--- + +## 2 · Our disclosure process 📜 + +1. **Triage** – confirm the issue, assess severity, assign CVSS v4 score. +2. **Patch development** – branch created in a private mirror; PoCs kept confidential. +3. **Pre‑notification** – downstream packagers & large adopters alerted **72 h** before release. +4. **Co‑ordinated release** – patched version + advisory (GHSA + CVE) + SBOM delta. +5. **Credits** – researchers listed in release notes (opt‑in). + +We aim for **30 days** from report to release for critical/high issues; medium/low may wait for the next scheduled release. + +--- + +## 3 · Existing safeguards ✅ + +| Layer | Control | +|-------|---------| +| **Release integrity** | `cosign` signatures + SPDX SBOM on every artefact | +| **Build pipeline** | Reproducible, fully declarative CI; SBOM diff verified in CI | +| **Runtime hardening** | Non‑root UID, distroless‑glibc base, SELinux/AppArmor profiles, cgroup CPU/RAM caps | +| **Access logs** | Retained **7 days**, then `sha256(ip)` hash | +| **Quota ledger** | Stores *token‑ID hash* only, no plain e‑mail/IP | +| **Air‑gap support** | Signed **Offline Update Kit** (OUK) validated before import | +| **Secure defaults** | TLS 1.3 (or stronger via plug‑in), HTTP Strict‑Transport‑Security, Content‑Security‑Policy | +| **SBOM re‑scan** | Nightly cron re‑checks previously “clean” images against fresh CVE feeds | + +--- + +## 4 · Cryptographic keys 🔑 + +| Purpose | Fingerprint | Where to fetch | +|---------|-------------|----------------| +| **PGP (sec‑team)** | `3A5C ​71F3 ​... ​7D9B` | [`/keys/#pgp`](../keys/#pgp) | +| **Cosign release key** | `AB12 ... EF90` | [`/keys/#cosign`](../keys/#cosign) | + +Verify all downloads (TLS 1.3 by default; 1.2 allowed only via a custom TLS provider such as GOST): + + +```bash +cosign verify \ + --key https://stella-ops.org/keys/cosign.pub \ + registry.stella-ops.org/stella-ops/stella-ops: +```` + +--- + +## 5 · Private‑feed mirrors 🌐 + +The **Feedser (vulnerability ingest/merge/export service)** provides signed JSON and Trivy DB snapshots that merge: + +* OSV + GHSA +* (optional) NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU regionals + +The snapshot ships in every Offline Update Kit and is validated with an in‑toto SLSA attestation at import time. + +--- + +## 6 · Hall of Thanks 🏆 + +We are grateful to the researchers who help keep Stella Ops safe: + +| Release | Researcher | Handle / Org | +| ------- | ------------------ | ------------ | +| *empty* | *(your name here)* | | + +--- diff --git a/docs/14_GLOSSARY_OF_TERMS.md b/docs/14_GLOSSARY_OF_TERMS.md new file mode 100755 index 00000000..300c0ade --- /dev/null +++ b/docs/14_GLOSSARY_OF_TERMS.md @@ -0,0 +1,112 @@ +# 14 · Glossary of Terms — Stella Ops + + +--- + +### 0 Purpose +A concise, single‑page **“what does that acronym actually mean?”** reference for +developers, DevOps engineers, IT managers and auditors who are new to the +Stella Ops documentation set. + +*If you meet a term in any Stella Ops doc that is **not** listed here, please +open a PR and append it alphabetically.* + +--- + +## A – C + +| Term | Short definition | Links / notes | +|------|------------------|---------------| +| **ADR** | *Architecture Decision Record* – lightweight Markdown file that captures one irreversible design decision. | ADR template lives at `/docs/adr/` | +| **AIRE** | *AI Risk Evaluator* – optional Plus/Pro plug‑in that suggests mute rules using an ONNX model. | Commercial feature | +| **Azure‑Pipelines** | CI/CD service in Microsoft Azure DevOps. | Recipe in Pipeline Library | +| **BDU** | Russian (FSTEC) national vulnerability database: *База данных уязвимостей*. | Merged with NVD by Feedser (vulnerability ingest/merge/export service) | +| **BuildKit** | Modern Docker build engine with caching and concurrency. | Needed for layer cache patterns | +| **CI** | *Continuous Integration* – automated build/test pipeline. | Stella integrates via CLI | +| **Cosign** | Open‑source Sigstore tool that signs & verifies container images **and files**. | Images & OUK tarballs | +| **CWV / CLS** | *Core Web Vitals* metric – Cumulative Layout Shift. | UI budget ≤ 0.1 | +| **CycloneDX** | Open SBOM (BOM) standard alternative to SPDX. | Planned report format plug‑in | + +--- + +## D – G + +| Term | Definition | Notes | +|------|------------|-------| +| **Digest (image)** | SHA‑256 hash uniquely identifying a container image or layer. | Pin digests for reproducible builds | +| **Docker‑in‑Docker (DinD)** | Running Docker daemon inside a CI container. | Used in GitHub / GitLab recipes | +| **DTO** | *Data Transfer Object* – C# record serialised to JSON. | Schemas in doc 11 | +| **Feedser** | Vulnerability ingest/merge/export service consolidating OVN, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds into the canonical MongoDB store and export artifacts. | Cron default `0 1 * * *` | +| **FSTEC** | Russian regulator issuing SOBIT certificates. | Pro GA target | +| **Gitea** | Self‑hosted Git service – mirrors GitHub repo. | OSS hosting | +| **GOST TLS** | TLS cipher‑suites defined by Russian GOST R 34.10‑2012 / 34.11‑2012. | Provided by `OpenSslGost` or CryptoPro | +| **Grype** | Alternative OSS vulnerability scanner; can be hot‑loaded as plug‑in. | Scanner interface `IScannerRunner` | + +--- + +## H – L + +| Term | Definition | Notes | +|------|------------|-------| +| **Helm** | Kubernetes package manager (charts). | Beta chart under `/charts/core` | +| **Hot‑load** | Runtime discovery & loading of plug‑ins **without restart**. | Cosign‑signed DLLs | +| **Hyperfine** | CLI micro‑benchmark tool used in Performance Workbook. | Outputs CSV | +| **JWT** | *JSON Web Token* – bearer auth token issued by OpenIddict. | Scope `scanner`, `admin`, `ui` | +| **K3s / RKE2** | Lightweight Kubernetes distributions (Rancher). | Supported in K8s guide | +| **Kubernetes NetworkPolicy** | K8s resource controlling pod traffic. | Redis/Mongo isolation | + +--- + +## M – O + +| Term | Definition | Notes | +|------|------------|-------| +| **Mongo (optional)** | Document DB storing > 180 day history and audit logs. | Off by default in Core | +| **Mute rule** | JSON object that suppresses specific CVEs until expiry. | Schema `mute-rule‑1.json` | +| **NVD** | US‑based *National Vulnerability Database*. | Primary CVE source | +| **ONNX** | Portable neural‑network model format; used by AIRE. | Runs in‑process | +| **OpenIddict** | .NET library that implements OAuth2 / OIDC in Stella backend. | Embedded IdP | +| **OUK** | *Offline Update Kit* – signed tarball with images + feeds for air‑gap. | Admin guide #24 | +| **OTLP** | *OpenTelemetry Protocol* – exporter for traces & metrics. | `/metrics` endpoint | + +--- + +## P – S + +| Term | Definition | Notes | +|------|------------|-------| +| **P95** | 95th‑percentile latency metric. | Target ≤ 5 s SBOM path | +| **PDF SAR** | *Security Assessment Report* PDF produced by Pro edition. | Cosign‑signed | +| **Plug‑in** | Hot‑loadable DLL implementing a Stella contract (`IScannerRunner`, `ITlsProvider`, etc.). | Signed with Cosign | +| **Problem Details** | RFC 7807 JSON error format returned by API. | See API ref §0 | +| **Redis** | In‑memory datastore used for queue + cache. | Port 6379 | +| **Rekor** | Sigstore transparency log; future work for signature anchoring. | Road‑map P4 | +| **RPS** | *Requests Per Second*. | Backend perf budget 40 rps | +| **SBOM** | *Software Bill of Materials* – inventory of packages in an image. | Trivy JSON v2 | +| **Stella CLI** | Lightweight CLI that submits SBOMs for vulnerability scanning. | See CI recipes | +| **Seccomp** | Linux syscall filter JSON profile. | Backend shipped non‑root | +| **SLA** | *Service‑Level Agreement* – 24 h / 1‑ticket for Pro. | SRE runbook | +| **Span** | .NET ref‑like struct for zero‑alloc slicing. | Allowed with benchmarks | +| **Styker.NET** | Mutation testing runner used on critical libs. | Coverage ≥ 60 % | + +--- + +## T – Z + +| Term | Definition | Notes | +|------|------------|-------| +| **Trivy** | OSS CVE scanner powering the default `IScannerRunner`. | CLI pinned 0.64 | +| **Trivy‑srv** | Long‑running Trivy server exposing gRPC API; speeds up remote scans. | Variant A | +| **UI tile** | Dashboard element showing live metric (scans today, feed age, etc.). | Angular Signals | +| **WebSocket** | Full‑duplex channel (`/ws/scan`, `/ws/stats`) for UI real‑time. | Used by tiles | +| **Zastava** | Lightweight agent that inventories running containers and can enforce kills. | | + +--- + +### 11 Change log + +| Version | Date | Notes | +|---------|------|-------| +| **v1.0** | 2025‑07‑12 | First populated glossary – 52 terms covering Core docs. | + +*(End of Glossary v1.0)* diff --git a/docs/15_UI_GUIDE.md b/docs/15_UI_GUIDE.md new file mode 100755 index 00000000..5537246c --- /dev/null +++ b/docs/15_UI_GUIDE.md @@ -0,0 +1,234 @@ +#  15 - Pragmatic UI Guide --- **Stella Ops** + +# Stella Ops Web UI + +A fast, modular single‑page application for controlling scans, policies, offline updates and platform‑wide settings. +Built for sub‑second feedback, dark‑mode by default, and **no external CDNs** – everything ships inside the anonymous internal registry. + +--- + +## 0 Fast Facts + +| Aspect | Detail | +| ----------------- | -------------------------------------------------------------------------- | +| Tech Stack | **Angular {{ angular }}** + Vite dev server | +| Styling | **Tailwind CSS** | +| State | Angular Signals + RxJS | +| API Client | OpenAPI v3 generated services (Axios) | +| Auth | OAuth2 /OIDC (tokens from backend or external IdP) | +| i18n | JSON bundles – **`/locales/{lang}.json`** (English, Russian shipped) | +| Offline Updates 📌 | UI supports “OUK” tarball upload to refresh NVD / Trivy DB when air‑gapped | +| Build Artifacts | (`ui/dist/`) pushed to `registry.git.stella-ops.org/ui:${SHA}` | + +--- + +## 1 Navigation Map + +``` +Dashboard +└─ Scans + ├─ Active + ├─ History + └─ Reports +└─ Policies 📌 + ├─ Editor (YAML / Rego) 📌 + ├─ Import / Export 📌 + └─ History +└─ Settings + ├─ SBOM Format 📌 + ├─ Registry 📌 + ├─ Offline Updates (OUK) 📌 + ├─ Themes (Light / Dark / System) 📌 + └─ Advanced +└─ Plugins 🛠 +└─ Help / About +``` + +*The **Offline Updates (OUK)** node under **Settings** is new.* + +--- + +## 2 Technology Overview + +### 2.1 Build & Deployment + +1. `npm i && npm build` → generates `dist/` (~2.1 MB gzip). +2. A CI job tags and pushes the artifact as `ui:${GIT_SHA}` to the internal registry. +3. Backend serves static assets from `/srv/ui` (mounted from the image layer). + +_No external fonts or JS – true offline guarantee._ + +### 2.2 Runtime Boot + +1. **AppConfigService** pulls `/api/v1/config/ui` (contains feature flags, default theme, enabled plugins). +2. Locale JSON fetched (`/locales/{lang}.json`, falls back to `en`). +3. Root router mounts lazy‑loaded **feature modules** in the order supplied by backend – this is how future route plugins inject pages without forking the UI. + +--- + +## 3 Feature Walk‑Throughs + +### 3.1 Dashboard – Real‑Time Status + +* **Δ‑SBOM heat‑map** 📌 shows how many scans used delta mode vs. full unpack. +* “Feed Age” tile turns **orange** if NVD feed is older than 24 h; reverts after an **OUK** upload 📌. +* Live WebSocket updates for scans in progress (SignalR channel). +* **Quota Tile** – shows **Scans Today / {{ quota_token }}**; turns yellow at **≤ 10% remaining** (≈ 90% used), + red at {{ quota_token }} . +* **Token Expiry Tile** – shows days left on *client.jwt* (offline only); + turns orange at < 7 days. + +### 3.2 Scans Module + +| View | What you can do | +| ----------- | ------------------------------------------------------------------------------------------------- | +| **Active** | Watch progress bar (ETA ≤ 5 s) – newly added **Format** and **Δ** badges appear beside each item. | +| **History** | Filter by repo, tag, policy result (pass/block/soft‑fail). | +| **Reports** | Click row → HTML or PDF report rendered by backend (`/report/{digest}/html`). | + +### 3.3 📌 Policies Module (new) + +*Embedded **Monaco** editor with YAML + Rego syntax highlighting.* + +| Tab | Capability | +| ------------------- | ------------------------------------------------------------------------------------------------ | +| **Editor** | Write or paste `scan-policy.yaml` or inline Rego snippet. Schema validation shown inline. | +| **Import / Export** | Buttons map to `/policy/import` and `/policy/export`. Accepts `.yaml`, `.rego`, `.zip` (bundle). | +| **History** | Immutable audit log; diff viewer highlights rule changes. | + +#### 3.3.1 YAML → Rego Bridge + +If you paste YAML but enable **Strict Mode** (toggle), backend converts to Rego under the hood, stores both representations, and shows a side‑by‑side diff. + +### 3.4 📌 Settings Enhancements + +| Setting | Details | +| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **SBOM Format** | Dropdown – *Trivy JSON*, *SPDX JSON*, *CycloneDX JSON*. | +| **Registry** | Displays pull URL (`registry.git.stella-ops.ru`) and Cosign key fingerprint. | +| **Offline Updates (OUK)** 📌 | Upload **`ouk*.tar.gz`** produced by the Offline Update Kit CLI. Backend unpacks, verifies SHA‑256 checksum & Cosign signature, then reloads Redis caches without restart. | +| **Theme** | Light, Dark, or Auto (system). | + +#### 3.4.1 OUK Upload Screen 📌 + +*Page path:* **Settings → Offline Updates (OUK)** +*Components:* + +1. **Drop Zone** – drag or select `.tar.gz` (max 1 GB). +2. **Progress Bar** – streaming upload with chunked HTTP. +3. **Verification Step** – backend returns status: + * *Signature valid* ✔️ + * *Digest mismatch* ❌ +4. **Feed Preview** – table shows *NVD date*, *OUI source build tag*, *CVE count delta*. +5. **Activate** – button issues `/feeds/activate/{id}`; on success the Dashboard “Feed Age” tile refreshes to green. +6. **History List** – previous OUK uploads with user, date, version; supports rollback. + +*All upload actions are recorded in the Policies → History audit log as type `ouk_update`.* + +### 3.5 Plugins Panel 🛠 (ships after UI modularisation) + +Lists discovered UI plugins; each can inject routes/panels. Toggle on/off without reload. + +### 3.6 Settings → **Quota & Tokens** (new) + +* View current **Client‑JWT claims** (tier, maxScansPerDay, expiry). +* **Generate Offline Token** – admin‑only button → POST `/token/offline` (UI wraps the API). +* Upload new token file for manual refresh. + +--- + +## 4 i18n & l10n + +* JSON files under `/locales`. +* Russian (`ru`) ships first‑class, translated security terms align with **GOST R ISO/IEC 27002‑2020**. +* “Offline Update Kit” surfaces as **“Оффлайн‑обновление базы уязвимостей”** in Russian locale. +* Community can add locales by uploading a new JSON via Plugins Panel once 🛠 ships. + +--- + +## 5 Accessibility + +* WCAG 2.1 AA conformance targeted. +* All color pairs pass contrast (checked by `vite-plugin-wcag`). +* Keyboard navigation fully supported; focus outlines visible in both themes. + +--- + +## 6 Theming 📌 + +| Layer | How to change | +| --------------- | ------------------------------------------------------------ | +| Tailwind | Palette variables under `tailwind.config.js > theme.colors`. | +| Runtime toggle | Stored in `localStorage.theme`, synced across tabs. | +| Plugin override | Future route plugins may expose additional palettes 🛠. | + +--- + +## 7 Extensibility Hooks + +| Area | Contract | Example | +| ------------- | ---------------------------------------- | ---------------------------------------------- | +| New route | `window.stella.registerRoute()` | “Secrets” scanner plugin adds `/secrets` page. | +| External link | `window.stella.addMenuLink(label, href)` | “Docs” link opens corporate Confluence. | +| Theme | `window.stella.registerTheme()` | High‑contrast palette for accessibility. | + +--- + +## 8 Road‑Map Tags + +| Feature | Status | +| ------------------------- | ------ | +| Policy Editor (YAML) | ✅ | +| Inline Rego validation | 🛠 | +| OUK Upload UI | ✅ | +| Plugin Marketplace UI | 🚧 | +| SLSA Verification banner | 🛠 | +| Rekor Transparency viewer | 🚧 | + +--- + +## 9 Non‑Commercial Usage Rules 📌 + +*(Extracted & harmonised from the Russian UI help page so that English docs remain licence‑complete.)* + +1. **Free for internal security assessments.** +2. Commercial resale or SaaS re‑hosting **prohibited without prior written consent** under AGPL §13. +3. If you distribute a fork **with UI modifications**, you **must**: + * Make the complete source code (including UI assets) publicly available. + * Retain original project attribution in footer. +4. All dependencies listed in `ui/package.json` remain under their respective OSS licences (MIT, Apache 2.0, ISC). +5. Use in government‑classified environments must comply with**applicable local regulations** governing cryptography and software distribution. + +--- + +## 10 Troubleshooting Tips + +| Symptom | Cause | Remedy | +| ----------------------------------- | ----------------------------------- | ----------------------------------------------------------------- | +| **White page** after login | `ui/dist/` hash mismatch | Clear browser cache; backend auto‑busts on version change. | +| Policy editor shows “Unknown field” | YAML schema drift | Sync your policy file to latest sample in *Settings → Templates*. | +| **OUK upload fails** at 99 % | Tarball built with outdated OUK CLI | Upgrade CLI (`ouk --version`) and rebuild package. | +| Icons look broken in Safari | *SVG `mask` unsupported* | Use Safari 17+ or switch to PNG icon set in Settings > Advanced. | + +--- + +## 11 Contributing + +* Run `npm dev` and open `http://localhost:5173`. +* Ensure `ng lint` and `ng test` pass before PR. +* Sign the **DCO** in your commit footer (`Signed-off-by`). + +--- + +## 12 Change Log + +| Version | Date | Highlights | +| ------- | ---------- | +| v2.4 | 2025‑07‑15 | **Added full OUK Offline Update upload flow** – navigation node, Settings panel, dashboard linkage, audit hooks. | +| v2.3 | 2025‑07‑14 | Added Policies module, SBOM Format & Registry settings, theming toggle, Δ‑SBOM indicators, extracted non‑commercial usage rules. | +| v2.2 | 2025‑07‑12 | Added user tips/workflows, CI notes, DevSecOps section, troubleshooting, screenshots placeholders. | +| v2.1 | 2025‑07‑12 | Removed PWA/Service‑worker; added oidc‑client‑ts; simplified roadmap | +| v2.0 | 2025‑07‑12 | Accessibility, Storybook, perf budgets, security rules | +| v1.1 | 2025‑07‑11 | Original OSS‑only guide | + +(End of Pragmatic UI Guide v2.2) diff --git a/docs/17_SECURITY_HARDENING_GUIDE.md b/docs/17_SECURITY_HARDENING_GUIDE.md new file mode 100755 index 00000000..068c01bb --- /dev/null +++ b/docs/17_SECURITY_HARDENING_GUIDE.md @@ -0,0 +1,186 @@ +#  17 · Security Hardening Guide — **Stella Ops** +*(v2.0 — 12 Jul 2025)* + +> **Audience** — Site‑reliability and platform teams deploying **the open‑source Core** in production or restricted networks. +--- + +##  0 Table of Contents + +1. Threat model (summary) +2. Host‑OS baseline +3. Container & runtime hardening +4. Network‑plane guidance +5. Secrets & key management +6. Image, SBOM & plug‑in supply‑chain controls +7. Logging, monitoring & audit +8. Update & patch strategy +9. Incident‑response workflow +10. Pen‑testing & continuous assurance +11. Contacts & vulnerability disclosure +12. Change log + +--- + +##  1 Threat model (summary) + +| Asset | Threats | Mitigations | +| -------------------- | --------------------- | ---------------------------------------------------------------------- | +| SBOMs & scan results | Disclosure, tamper | TLS‑in‑transit, read‑only Redis volume, RBAC, Cosign‑verified plug‑ins | +| Backend container | RCE, code‑injection | Distroless image, non‑root UID, read‑only FS, seccomp + `CAP_DROP:ALL` | +| Update artefacts | Supply‑chain attack | Cosign‑signed images & SBOMs, enforced by admission controller | +| Admin credentials | Phishing, brute force | OAuth 2.0 with 12‑h token TTL, optional mTLS | + +--- + +##  2 Host‑OS baseline checklist + +| Item | Recommended setting | +| ------------- | --------------------------------------------------------- | +| OS | Ubuntu 22.04 LTS (kernel ≥ 5.15) or Alma 9 | +| Patches | `unattended‑upgrades` or vendor‑equivalent enabled | +| Filesystem | `noexec,nosuid` on `/tmp`, `/var/tmp` | +| Docker Engine | v24.*, API socket root‑owned (`0660`) | +| Auditd | Watch `/etc/docker`, `/usr/bin/docker*` and Compose files | +| Time sync | `chrony` or `systemd‑timesyncd` | + +--- + +##  3 Container & runtime hardening + +###  3.1 Docker Compose reference (`compose-core.yml`) + +```yaml +services: + backend: + image: registry.stella-ops.org/stella-ops/stella-ops: + user: "101:101" # non‑root + read_only: true + security_opt: + - "no-new-privileges:true" + - "seccomp:./seccomp-backend.json" + cap_drop: [ALL] + tmpfs: + - /tmp:size=64m,exec,nosymlink + environment: + - ASPNETCORE_URLS=https://+:8080 + - TLSPROVIDER=OpenSslGost + depends_on: [redis] + networks: [core-net] + healthcheck: + test: ["CMD", "wget", "-qO-", "https://localhost:8080/health"] + interval: 30s + timeout: 5s + retries: 5 + + redis: + image: redis:7.2-alpine + command: ["redis-server", "--requirepass", "${REDIS_PASS}", "--rename-command", "FLUSHALL", ""] + user: "redis" + read_only: true + cap_drop: [ALL] + tmpfs: + - /data + networks: [core-net] + +networks: + core-net: + driver: bridge +``` + +No dedicated “Redis” or “Mongo” sub‑nets are declared; the single bridge network suffices for the default stack. + +###  3.2 Kubernetes deployment highlights + +Use a separate NetworkPolicy that only allows egress from backend to Redis :6379. +securityContext: runAsNonRoot, readOnlyRootFilesystem, allowPrivilegeEscalation: false, drop all capabilities. +PodDisruptionBudget of minAvailable: 1. +Optionally add CosignVerified=true label enforced by an admission controller (e.g. Kyverno or Connaisseur). + +##  4 Network‑plane guidance + +| Plane | Recommendation | +| ------------------ | -------------------------------------------------------------------------- | +| North‑south | Terminate TLS 1.2+ (OpenSSL‑GOST default). Use LetsEncrypt or internal CA. | +| East‑west | Compose bridge or K8s ClusterIP only; no public Redis/Mongo ports. | +| Ingress controller | Limit methods to GET, POST, PATCH (no TRACE). | +| Rate‑limits | 40 rps default; tune ScannerPool.Workers and ingress limit‑req to match. | + +##  5 Secrets & key management + +| Secret | Storage | Rotation | +| --------------------------------- | ---------------------------------- | ----------------------------- | +| **Client‑JWT (offline)** | `/var/lib/stella/tokens/client.jwt` (root : 600) | **30 days** – provided by each OUK | +| REDIS_PASS | Docker/K8s secret | 90 days | +| OAuth signing key | /keys/jwt.pem (read‑only mount) | 180 days | +| Cosign public key | /keys/cosign.pub baked into image; | change on every major release | +| Trivy DB mirror token (if remote) | Secret + read‑only | 30 days | + +Never bake secrets into images; always inject at runtime. + +> **Operational tip:** schedule a cron reminding ops 5 days before +> `client.jwt` expiry. The backend also emits a Prometheus metric +> `stella_quota_token_days_remaining`. + +##  6 Image, SBOM & plug‑in supply‑chain controls + +* Images — Pull by digest not latest; verify: + +```bash +cosign verify ghcr.io/stellaops/backend@sha256: \ + --key https://stella-ops.org/keys/cosign.pub +``` + +* SBOM — Each release ships an SPDX file; store alongside images for audit. +* Third‑party plug‑ins — Place in /plugins/; backend will: +* Validate Cosign signature. +* Check [StellaPluginVersion("major.minor")]. +* Refuse to start if Security.DisablePluginUnsigned=false (default). + +##  7 Logging, monitoring & audit + +| Control | Implementation | +| ------------ | ----------------------------------------------------------------- | +| Log format | Serilog JSON; ship via Fluent‑Bit to ELK or Loki | +| Metrics | Prometheus /metrics endpoint; default Grafana dashboard in infra/ | +| Audit events | Redis stream audit; export daily to SIEM | +| Alert rules | Feed age  ≥ 48 h, P95 wall‑time > 5 s, Redis used memory > 75 % | + +##  8 Update & patch strategy + +| Layer | Cadence | Method | +| -------------------- | -------------------------------------------------------- | ------------------------------ | +| Backend & CLI images | Monthly or CVE‑driven docker pull + docker compose up -d | +| Trivy DB | 24 h scheduler via Feedser (vulnerability ingest/merge/export service) | configurable via Feedser scheduler options | +| Docker Engine | vendor LTS | distro package manager | +| Host OS | security repos enabled | unattended‑upgrades | + +##  9 Incident‑response workflow + +* Detect — PagerDuty alert from Prometheus or SIEM. +* Contain — Stop affected Backend container; isolate Redis RDB snapshot. +* Eradicate — Pull verified images, redeploy, rotate secrets. +* Recover — Restore RDB, replay SBOMs if history lost. +* Review — Post‑mortem within 72 h; create follow‑up issues. +* Escalate P1 incidents to (24 × 7). + + +##  10 Pen‑testing & continuous assurance + +| Control | Frequency | Tool/Runner | +|----------------------|-----------------------|-------------------------------------------| +| OWASP ZAP baseline | Each merge to `main` | GitHub Action `zap-baseline-scan` | +| Dependency scanning | Per pull request | Trivy FS + Dependabot | +| External red‑team | Annual or pre‑GA | CREST‑accredited third‑party | + +##  11 Vulnerability disclosure & contact + +* Preferred channel: security@stella‑ops.org (GPG key on website). +* Coordinated disclosure reward: public credit and swag (no monetary bounty at this time). + +##  12 Change log + +| Version | Date | Notes | +| ------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------- | +| v2.0 | 2025‑07‑12 | Full overhaul: host‑OS baseline, supply‑chain signing, removal of unnecessary sub‑nets, role‑based contact e‑mail, K8s guidance. | +| v1.1 | 2025‑07‑09 | Minor fence fixes. | +| v1.0 | 2025‑07‑09 | Original draft. | diff --git a/docs/18_CODING_STANDARDS.md b/docs/18_CODING_STANDARDS.md new file mode 100755 index 00000000..50c28af5 --- /dev/null +++ b/docs/18_CODING_STANDARDS.md @@ -0,0 +1,169 @@ +#  18 · Coding Standards & Contributor Guide — **Stella Ops** +*(v2.0 — 12 Jul 2025 · supersedes v1.0)* + +> **Audience** — Anyone sending a pull‑request to the open‑source Core. +> **Goal** — Keep the code‑base small‑filed, plug‑in‑friendly, DI‑consistent, and instantly readable. + +--- + +##  0 Why read this? + +* Cuts review time → quicker merges. +* Guarantees code is **hot‑load‑safe** for run‑time plug‑ins. +* Prevents style churn and merge conflicts. + +--- + +##  1 High‑level principles + +1. **SOLID first** – especially Interface & Dependency Inversion. +2. **100‑line rule** – any file > 100 physical lines must be split or refactored. +3. **Contract‑level ownership** – public abstractions live in lightweight *Contracts* libraries; impl classes live in runtime projects. +4. **Single Composition Root** – all DI wiring happens in **`StellaOps.Web/Program.cs`** and in each plug‑in’s `IoCConfigurator`; nothing else calls `IServiceCollection.BuildServiceProvider`. +5. **No Service Locator** – constructor injection only; static `ServiceProvider` is banned. +6. **Fail‑fast startup** – configuration validated before the web‑host listens. +7. **Hot‑load compatible** – no static singletons that survive plug‑in unload; avoid `Assembly.LoadFrom` outside the built‑in plug‑in loader. + +--- + +##  2 Repository layout (flat, July‑2025)** + +```text +src/ +├─ backend/ +│ ├─ StellaOps.Web/ # ASP.NET host + composition root +│ ├─ StellaOps.Common/ # Serilog, Result, helpers +│ ├─ StellaOps.Contracts/ # DTO + interface contracts (no impl) +│ ├─ StellaOps.Configuration/ # Options + validation +│ ├─ StellaOps.Localization/ +│ ├─ StellaOps.PluginLoader/ # Cosign verify, hot‑load +│ ├─ StellaOps.Scanners.Trivy/ # First‑party scanner +│ ├─ StellaOps.TlsProviders.OpenSsl/ +│ └─ … (additional runtime projects) +├─ plugins-sdk/ # Templated contracts & abstractions +└─ frontend/ # Angular workspace +tests/ # Mirrors src structure 1‑to‑1 +``` + +There are no folders named “Module” and no nested solutions. + +##  3 Naming & style conventions + +| Element | Rule | Example | +| ------------------------------------------------------------------------------- | --------------------------------------- | ------------------------------- | +| Namespaces | File‑scoped, StellaOps. | namespace StellaOps.Scanners; | +| Interfaces | I prefix, PascalCase | IScannerRunner | +| Classes / records | PascalCase | ScanRequest, TrivyRunner | +| Private fields | camelCase (no leading underscore) | redisCache, httpClient | +| Constants | SCREAMING_SNAKE_CASE | const int MAX_RETRIES = 3; | +| Async methods | End with Async | Task ScanAsync() | +| File length | ≤ 100 lines incl. using & braces | enforced by dotnet format check | +| Using directives | Outside namespace, sorted, no wildcards | — | + +Static analyzers (.editorconfig, StyleCop.Analyzers package) enforce the above. + +##  4 Dependency‑injection policy + +Composition root – exactly one per process: + +```csharp +builder.Services + .AddStellaCore() // extension methods from each runtime project + .AddPluginLoader("/Plugins", cfg); // hot‑load signed DLLs +``` + +Plug‑ins register additional services via the IoCConfigurator convention described in the Plug‑in SDK Guide, §5. +Never resolve services manually (provider.GetService()) outside the composition root; tests may use WebApplicationFactory or ServiceProvider.New() helpers. +Scoped lifetime is default; singletons only for stateless, thread‑safe helpers. + +##  5 Project organisation rules + +Contracts vs. Runtime – public DTO & interfaces live in .Contracts; implementation lives in sibling project. +Feature folders – inside each runtime project group classes by use‑case, e.g. + +```text +├─ Scan/ +│ ├─ ScanService.cs +│ └─ ScanController.cs +├─ Feed/ +└─ Tls/ +``` + +Tests – mirror the structure under tests/ one‑to‑one; no test code inside production projects. + +##  6 C# language features + +Nullable reference types enabled. +record for immutable DTOs. +Pattern matching encouraged; avoid long switch‑cascades. +Span & Memory OK when perf‑critical, but measure first. +Use await foreach over manual paginator loops. + +##  7 Error‑handling template + +```csharp +public async Task PostScan([FromBody] ScanRequest req) +{ + if (!ModelState.IsValid) return BadRequest(ModelState); + + try + { + ScanResult result = await scanService.ScanAsync(req); + if (result.Quota != null) + { + Response.Headers.TryAdd("X-Stella-Quota-Remaining", result.Quota.Remaining.ToString()); + Response.Headers.TryAdd("X-Stella-Reset", result.Quota.ResetUtc.ToString("o")); + } + return Ok(result); + } +} +``` + +RFC 7807 ProblemDetails for all non‑200s. +Capture structured logs with Serilog’s message‑template syntax. + +##  8 Async & threading + +* All I/O is async; no .Result / .Wait(). +* Library code: ConfigureAwait(false). +* Limit concurrency via Channel or Parallel.ForEachAsync, never raw Task.Run loops. + +##  9 Testing rules + +| Layer | Framework | Coverage gate | +| ------------------------ | ------------------------ | -------------------------- | +| Unit | xUnit + FluentAssertions | ≥ 80 % line, ≥ 60 % branch | +| Integration | Testcontainers | Real Redis & Trivy | +| Mutation (critical libs) | Stryker.NET | ≥ 60 % score | + +One test project per runtime/contract project; naming .Tests. + +##  10 Static analysis & formatting + +* dotnet format must exit clean (CI gate). +* StyleCop.Analyzers + Roslyn‑Security‑Guard run on every PR. +* CodeQL workflow runs nightly on main. + +##  11 Commit & PR checklist + +* Conventional Commit prefix (feat:, fix:, etc.). +* dotnet format & dotnet test both green. +* Added or updated XML‑doc comments for public APIs. +* File count & length comply with 100‑line rule. +* If new public contract → update relevant markdown doc & JSON‑Schema. + +##  12 Common pitfalls + +|Symptom| Root cause | Fix +|-------|-------------|------------------- +|InvalidOperationException: Cannot consume scoped service...| Mis‑matched DI lifetimes| Use scoped everywhere unless truly stateless +|Hot‑reload plug‑in crash| Static singleton caching plugin types| Store nothing static; rely on DI scopes + +> 100‑line style violation |Large handlers or utils |Split into private helpers or new class + +##  13 Change log + +| Version | Date | Notes | +| ------- | ---------- | -------------------------------------------------------------------------------------------------- | +| v2.0 | 2025‑07‑12 | Updated DI policy, 100‑line rule, new repo layout, camelCase fields, removed “Module” terminology. | +| 1.0 | 2025‑07‑09 | Original standards. | diff --git a/docs/19_TEST_SUITE_OVERVIEW.md b/docs/19_TEST_SUITE_OVERVIEW.md new file mode 100755 index 00000000..b4208672 --- /dev/null +++ b/docs/19_TEST_SUITE_OVERVIEW.md @@ -0,0 +1,91 @@ +# Automated Test‑Suite Overview + +This document enumerates **every automated check** executed by the Stella Ops +CI pipeline, from unit level to chaos experiments. It is intended for +contributors who need to extend coverage or diagnose failures. + +> **Build parameters** – values such as `{{ dotnet }}` (runtime) and +> `{{ angular }}` (UI framework) are injected at build time. + +--- + +## Layer map + +| Layer | Tooling | Entry‑point | Frequency | +|-------|---------|-------------|-----------| +| **1. Unit** | `xUnit` (dotnet test) | `*.Tests.csproj` | per PR / push | +| **2. Property‑based** | `FsCheck` | `SbomPropertyTests` | per PR | +| **3. Integration (API)** | `Testcontainers` suite | `test/Api.Integration` | per PR + nightly | +| **4. Integration (DB-merge)** | in-memory Mongo + Redis | `Feedser.Integration` (vulnerability ingest/merge/export service) | per PR | +| **5. Contract (gRPC)** | `Buf breaking` | `buf.yaml` files | per PR | +| **6. Front‑end unit** | `Jest` | `ui/src/**/*.spec.ts` | per PR | +| **7. Front‑end E2E** | `Playwright` | `ui/e2e/**` | nightly | +| **8. Lighthouse perf / a11y** | `lighthouse-ci` (Chrome headless) | `ui/dist/index.html` | nightly | +| **9. Load** | `k6` scripted scenarios | `k6/*.js` | nightly | +| **10. Chaos CPU / OOM** | `pumba` | Docker Compose overlay | weekly | +| **11. Dependency scanning** | `Trivy fs` + `dotnet list package --vuln` | root | per PR | +| **12. License compliance** | `LicenceFinder` | root | per PR | +| **13. SBOM reproducibility** | `in‑toto attestation` diff | GitLab job | release tags | + +--- + +## Quality gates + +| Metric | Budget | Gate | +|--------|--------|------| +| API unit coverage | ≥ 85 % lines | PR merge | +| API response P95 | ≤ 120 ms | nightly alert | +| Δ‑SBOM warm scan P95 (4 vCPU) | ≤ 5 s | nightly alert | +| Lighthouse performance score | ≥ 90 | nightly alert | +| Lighthouse accessibility score | ≥ 95 | nightly alert | +| k6 sustained RPS drop | < 5 % vs baseline | nightly alert | + +--- + +## Local runner + +```bash +# minimal run: unit + property + frontend tests +./scripts/dev-test.sh + +# full stack incl. Playwright and lighthouse +./scripts/dev-test.sh --full +```` + +The script spins up MongoDB/Redis via Testcontainers and requires: + +* Docker ≥ 25 +* Node 20 (for Jest/Playwright) + +--- + +## CI job layout + +```mermaid +flowchart LR + subgraph fast-path + U[xUnit] --> P[FsCheck] --> I1[Testcontainer API] + end + + I1 --> FE[Jest] + FE --> E2E[Playwright] + E2E --> Lighthouse + Lighthouse --> INTEG2[Feedser] + INTEG2 --> LOAD[k6] + LOAD --> CHAOS[pumba] + CHAOS --> RELEASE[Attestation diff] +``` + +--- + +## Adding a new test layer + +1. Extend `scripts/dev-test.sh` so local contributors get the layer by default. +2. Add a dedicated GitLab job in `.gitlab-ci.yml` (stage `test` or `nightly`). +3. Register the job in `docs/19_TEST_SUITE_OVERVIEW.md` *and* list its metric + in `docs/metrics/README.md`. + +--- + +*Last updated {{ "now" | date: "%Y‑%m‑%d" }}* + diff --git a/docs/21_INSTALL_GUIDE.md b/docs/21_INSTALL_GUIDE.md new file mode 100755 index 00000000..7b98b51d --- /dev/null +++ b/docs/21_INSTALL_GUIDE.md @@ -0,0 +1,131 @@ +# Stella Ops — Installation Guide (Docker & Air‑Gap) + + + +> **Status — public α not yet published.** +> The commands below will work as soon as the first image is tagged +> `registry.stella-ops.org/stella-ops/stella-ops:0.1.0-alpha` +> (target date: **late 2025**). Track progress on the +> [road‑map](/roadmap/). + +--- + +## 0 · Prerequisites + +| Item | Minimum | Notes | +|------|---------|-------| +| Linux | Ubuntu 22.04 LTS / Alma 9 | x86‑64 or arm64 | +| CPU / RAM | 2 vCPU / 2 GiB | Laptop baseline | +| Disk | 10 GiB SSD | SBOM + vuln DB cache | +| Docker | **Engine 25 + Compose v2** | `docker -v` | +| TLS | OpenSSL 1.1 +  | Self‑signed cert generated at first run | + +--- + +## 1 · Connected‑host install (Docker Compose) + +```bash +# 1. Make a working directory +mkdir stella && cd stella + +# 2. Download the signed Compose bundle + example .env +curl -LO https://get.stella-ops.org/releases/latest/.env.example +curl -LO https://get.stella-ops.org/releases/latest/.env.example.sig +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.infrastructure.yml +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.infrastructure.yml.sig +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.stella-ops.yml +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.stella-ops.yml.sig + +# 3. Verify provenance (Cosign public key is stable) +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature .env.example.sig \ + .env.example + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.infrastructure.yml.sig \ + docker-compose.infrastructure.yml + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.stella-ops.yml.sig \ + docker-compose.stella-ops.yml + +# 4. Copy .env.example → .env and edit secrets +cp .env.example .env +$EDITOR .env + +# 5. Launch databases (MongoDB + Redis) +docker compose --env-file .env -f docker-compose.infrastructure.yml up -d + +# 6. Launch Stella Ops (first run pulls ~50 MB merged vuln DB) +docker compose --env-file .env -f docker-compose.stella-ops.yml up -d +```` + +*Default login:* `admin / changeme` +UI: [https://\<host\>:8443](https://<host>:8443) (self‑signed certificate) + +> **Pinning best‑practice** – in production environments replace +> `stella-ops:latest` with the immutable digest printed by +> `docker images --digests`. + +--- + +## 2 · Optional: request a free quota token + +Anonymous installs allow **{{ quota\_anon }} scans per UTC day**. +Email `token@stella-ops.org` to receive a signed JWT that raises the limit to +**{{ quota\_token }} scans/day**. Insert it into `.env`: + +```bash +STELLA_JWT="paste‑token‑here" +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella-ops stella set-jwt "$STELLA_JWT" +``` + +>  The UI shows a reminder at 200 scans and throttles above the limit but will +>  **never block** your pipeline. + +--- + +## 3 · Air‑gapped install (Offline Update Kit) + +When running on an isolated network use the **Offline Update Kit (OUK)**: + +```bash +# Download & verify on a connected host +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-v0.1a.tgz +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-v0.1a.tgz.sig + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-kit-v0.1a.tgz.sig \ + stella-ops-offline-kit-v0.1a.tgz + +# Transfer → air‑gap → import +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella admin import-offline-usage-kit stella-ops-offline-kit-v0.1a.tgz +``` + +*Import is atomic; no service downtime.* + +For details see the dedicated [Offline Kit guide](/offline/). + +--- + +## 4 · Next steps + +* **5‑min Quick‑Start:** `/quickstart/` +* **CI recipes:** `docs/ci/20_CI_RECIPES.md` +* **Plug‑in SDK:** `/plugins/` + +--- + +*Generated {{ "now" | date: "%Y‑%m‑%d" }} — build tags inserted at render time.* diff --git a/docs/23_FAQ_MATRIX.md b/docs/23_FAQ_MATRIX.md new file mode 100755 index 00000000..c4636538 --- /dev/null +++ b/docs/23_FAQ_MATRIX.md @@ -0,0 +1,61 @@ +# Stella Ops — Frequently Asked Questions (Matrix) + +## Quick glance + +| Question | Short answer | +|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| What is Stella Ops? | A lightning‑fast, SBOM‑first container‑security scanner written in **.NET {{ dotnet }}** with an **Angular {{ angular }}** web UI. | +| How fast is it? | Warm scans finish in **\< 5 s** on a 4‑vCPU runner; first scans stay **\< 30 s**. | +| Is it free? | Yes – **{{ quota_anon }} scans / day** anonymously. Requesting a free JWT lifts the limit to **{{ quota_token }}**. A gentle reminder shows at 200; exceeding the cap throttles speed but never blocks. | +| Does it run offline? | Yes — download the signed **Offline Update Kit**; see `/offline/`. | +| Can I extend it? | Yes — restart‑time plug‑ins (`ISbomMutator`, `IVulnerabilityProvider`, `IResultSink`, OPA Rego). Marketplace GA in v1.0. | + +--- + +## Road‑map (authoritative link) + +The full, always‑up‑to‑date roadmap lives at . +Snapshot: + +| Version | Target date | Locked‑in scope (freeze at β) | +|---------|-------------|--------------------------------| +| **v0.1 α** | *Late 2025* | Δ‑SBOM engine, nightly re‑scan, Offline Kit v1, {{ quota_anon }}/ {{ quota_token }} quota | +| **v0.2 β** | Q1 2026 | *Zastava* forbidden‑image scanner, registry sweeper, SDK β | +| **v0.3 β** | Q2 2026 | YAML/Rego policy‑as‑code, SARIF output, OUK auto‑import | +| **v0.4 RC** | Q3 2026 | AI remediation advisor, LDAP/AD SSO, pluggable TLS providers | +| **v1.0 GA** | Q4 2026 | SLSA L3 provenance, signed plug‑in marketplace | + +--- + +## Technical matrix + +| Category | Detail | +|----------|--------| +| **Core runtime** | C# 14 on **.NET {{ dotnet }}** | +| **UI stack** | **Angular {{ angular }}** + TailwindCSS | +| **Container base** | Distroless glibc (x86‑64 & arm64) | +| **Data stores** | MongoDB 7 (SBOM + findings), Redis 7 (LRU cache + quota) | +| **Release integrity** | Cosign‑signed images & TGZ, reproducible build, SPDX 2.3 SBOM | +| **Extensibility** | Plug‑ins in any .NET language (restart load); OPA Rego policies | +| **Default quotas** | Anonymous **{{ quota_anon }} scans/day** · JWT **{{ quota_token }}** | + +--- + +## Quota enforcement (overview) + +* Counters live in Redis with 24 h keys: `quota:ip:` or `quota:tid:`. +* Soft reminder banner at 200 daily scans. +* Past the limit: first 30 excess requests delayed 5 s; afterwards 60 s. +* Behaviour is identical online and offline (validation local). + +For full flow see `docs/30_QUOTA_ENFORCEMENT_FLOW1.md`. + +--- + +## Further reading + +* **Install guide:** `/install/` +* **Offline mode:** `/offline/` +* **Security policy:** `/security/` +* **Governance:** `/governance/` +* **Community chat:** Matrix `#stellaops:libera.chat` diff --git a/docs/24_OFFLINE_KIT.md b/docs/24_OFFLINE_KIT.md new file mode 100755 index 00000000..156beb90 --- /dev/null +++ b/docs/24_OFFLINE_KIT.md @@ -0,0 +1,94 @@ +# Offline Update Kit (OUK) — Air‑Gap Bundle + + + +The **Offline Update Kit** packages everything Stella Ops needs to run on a +completely isolated network: + +| Component | Contents | +|-----------|----------| +| **Merged vulnerability feeds** | OSV, GHSA plus optional NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU | +| **Container images** | `stella-ops`, *Zastava* sidecar (x86‑64 & arm64) | +| **Provenance** | Cosign signature, SPDX 2.3 SBOM, in‑toto SLSA attestation | +| **Delta patches** | Daily diff bundles keep size \< 350 MB | + +*Scanner core:* C# 12 on **.NET {{ dotnet }}**. +*Imports are idempotent and atomic — no service downtime.* + +--- + +## 1 · Download & verify + +```bash +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-.tgz +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-.tgz.sig + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-kit-.tgz.sig \ + stella-ops-offline-kit-.tgz +```` + +Verification prints **OK** and the SHA‑256 digest; cross‑check against the +[changelog](https://git.stella-ops.org/stella-ops/offline-kit/-/releases). + +--- + +## 2 · Import on the air‑gapped host + +```bash +docker compose --env-file .env \ + -f docker-compose.stella-ops.yml \ + exec stella-ops \ + stella admin import-offline-usage-kit stella-ops-offline-kit-.tgz +``` + +* The CLI validates the Cosign signature **before** activation. +* Old feeds are kept until the new bundle is fully verified. +* Import time on a SATA SSD: ≈ 25 s for a 300 MB kit. + +--- + +## 3 · Delta patch workflow + +1. **Connected site** fetches `stella-ouk-YYYY‑MM‑DD.delta.tgz`. +2. Transfer via any medium (USB, portable disk). +3. `stella admin import-offline-usage-kit ` applies only changed CVE rows & images. + +Daily deltas are **< 30 MB**; weekly roll‑up produces a fresh full kit. + +--- + +## 4 · Quota behaviour offline + +The scanner enforces the same fair‑use limits offline: + +* **Anonymous:** {{ quota\_anon }} scans per UTC day +* **Free JWT:** {{ quota\_token }} scans per UTC day + +Soft reminder at 200 scans; throttle above the ceiling but **never block**. +See the detailed rules in +[`33_333_QUOTA_OVERVIEW.md`](33_333_QUOTA_OVERVIEW.md). + +--- + +## 5 · Troubleshooting + +| Symptom | Explanation | Fix | +| -------------------------------------- | ---------------------------------------- | ------------------------------------- | +| `could not verify SBOM hash` | Bundle corrupted in transit | Re‑download / re‑copy | +| Import hangs at `Applying feeds…` | Low disk space in `/var/lib/stella` | Free ≥ 2 GiB before retry | +| `quota exceeded` same day after import | Import resets counters at UTC 00:00 only | Wait until next UTC day or load a JWT | + +--- + +## 6 · Related documentation + +* **Install guide:** `/install/#air-gapped` +* **Sovereign mode rationale:** `/sovereign/` +* **Security policy:** `/security/#reporting-a-vulnerability` diff --git a/docs/29_LEGAL_FAQ_QUOTA.md b/docs/29_LEGAL_FAQ_QUOTA.md new file mode 100755 index 00000000..28a69c5b --- /dev/null +++ b/docs/29_LEGAL_FAQ_QUOTA.md @@ -0,0 +1,84 @@ +# Legal FAQ — Free‑Tier Quota & AGPL Compliance + +> **Operational behaviour (limits, counters, delays) is documented in +> [`33_333_QUOTA_OVERVIEW.md`](33_333_QUOTA_OVERVIEW.md).** +> This page covers only the legal aspects of offering Stella Ops as a +> service or embedding it into another product while the free‑tier limits are +> in place. + +--- + +## 1 · Does enforcing a quota violate the AGPL? + +**No.** +AGPL‑3.0 does not forbid implementing usage controls in the program itself. +Recipients retain the freedoms to run, study, modify and share the software. +The Stella Ops quota: + +* Is enforced **solely at the service layer** (Redis counters) — the source + code implementing the quota is published under AGPL‑3.0‑or‑later. +* Never disables functionality; it introduces *time delays* only after the + free allocation is exhausted. +* Can be bypassed entirely by rebuilding from source and removing the + enforcement middleware — the licence explicitly allows such modifications. + +Therefore the quota complies with §§ 0 & 2 of the AGPL. + +--- + +## 2 · Can I redistribute Stella Ops with the quota removed? + +Yes, provided you: + +1. **Publish the full corresponding source code** of your modified version + (AGPL § 13 & § 5c), and +2. Clearly indicate the changes (AGPL § 5a). + +You may *retain* or *relax* the limits, or introduce your own tiering, as long +as the complete modified source is offered to every user of the service. + +--- + +## 3 · Embedding in a proprietary appliance + +You may ship Stella Ops inside a hardware or virtual appliance **only if** the +entire combined work is distributed under **AGPL‑3.0‑or‑later** and you supply +the full source code for both the scanner and your integration glue. + +Shipping an AGPL component while keeping the rest closed‑source violates +§ 13 (*“remote network interaction”*). + +--- + +## 4 · SaaS redistribution + +Operating a public SaaS that offers Stella Ops scans to third parties triggers +the **network‑use clause**. You must: + +* Provide the complete, buildable source of **your running version** — + including quota patches or UI branding. +* Present the offer **conspicuously** (e.g. a “Source Code” footer link). + +Failure to do so breaches § 13 and can terminate your licence under § 8. + +--- + +## 5 · Is e‑mail collection for the JWT legal? + +* **Purpose limitation (GDPR Art. 5‑1 b):** address is used only to deliver the + JWT or optional release notes. +* **Data minimisation (Art. 5‑1 c):** no name, IP or marketing preferences are + required; a blank e‑mail body suffices. +* **Storage limitation (Art. 5‑1 e):** addresses are deleted or hashed after + ≤ 7 days unless the sender opts into updates. + +Hence the token workflow adheres to GDPR principles. + +--- + +## 6 · Change‑log + +| Version | Date | Notes | +|---------|------|-------| +| **2.0** | 2025‑07‑16 | Removed runtime quota details; linked to new authoritative overview. | +| 1.0 | 2024‑12‑20 | Initial legal FAQ. | diff --git a/docs/30_QUOTA_ENFORCEMENT_FLOW1.md b/docs/30_QUOTA_ENFORCEMENT_FLOW1.md new file mode 100755 index 00000000..480753f7 --- /dev/null +++ b/docs/30_QUOTA_ENFORCEMENT_FLOW1.md @@ -0,0 +1,93 @@ +# Quota Enforcement — Flow Diagram (rev 2.1) + +> **Scope** – this document explains *how* the free‑tier limits are enforced +> inside the scanner service. For policy rationale and legal aspects see +> [`33_333_QUOTA_OVERVIEW.md`](33_333_QUOTA_OVERVIEW.md). + +--- + +## 0 · Key parameters (rev 2.1) + +| Symbol | Value | Meaning | +|--------|-------|---------| +| `L_anon` | **{{ quota_anon }}** | Daily ceiling for anonymous users | +| `L_jwt` | **{{ quota_token }}** | Daily ceiling for token holders | +| `T_warn` | `200` | Soft reminder threshold | +| `D_soft` | `5 000 ms` | Delay for *first 30* over‑quota scans | +| `D_hard` | `60 000 ms` | Delay for all scans beyond the soft window | + +`L_active` is `L_jwt` if a valid token is present; else `L_anon`. + +--- + +## 1 · Sequence diagram + +```mermaid +sequenceDiagram + participant C as Client + participant API as Scanner API + participant REDIS as Redis (quota) + C->>API: /scan + API->>REDIS: INCR quota: + REDIS-->>API: new_count + alt new_count ≤ L_active + API-->>C: 202 Accepted (no delay) + else new_count ≤ L_active + 30 + API->>C: wait D_soft + API-->>C: 202 Accepted + else + API->>C: wait D_hard + API-->>C: 202 Accepted + end +```` + +*Counters auto‑expire **24 h** after first increment (00:00 UTC reset).* + +--- + +## 2 · Redis key layout + +| Key pattern | TTL | Description | +| ---------------------- | ---- | --------------------------------- | +| `quota:ip:` | 24 h | Anonymous quota per *hashed* IP | +| `quota:tid:` | 24 h | Token quota per *hashed* token‑ID | +| `quota:ip::ts` | 24 h | First‑seen timestamp (ISO 8601) | + +Keys share a common TTL for efficient mass expiry via `redis-cli --scan`. + +--- + +## 3 · Pseudocode (Go‑style) + +```go +func gate(key string, limit int) (delay time.Duration) { + cnt, _ := rdb.Incr(ctx, key).Result() + + switch { + case cnt <= limit: + return 0 // under quota + case cnt <= limit+30: + return 5 * time.Second + default: + return 60 * time.Second + } +} +``` + +*The middleware applies `time.Sleep(delay)` **before** processing the scan +request; it never returns `HTTP 429` under the free tier.* + +--- + +## 4 · Metrics & monitoring + +| Metric | PromQL sample | Alert | +| ------------------------------ | ------------------------------------------ | --------------------- | +| `stella_quota_soft_hits_total` | `increase(...[5m]) > 50` | Many users near limit | +| `stella_quota_hard_hits_total` | `rate(...[1h]) > 0.1` | Potential abuse | +| Average delay per request | `histogram_quantile(0.95, sum(rate(...)))` | P95 < 1 s expected | + +--- + + +*Generated {{ "now" | date: "%Y‑%m‑%d" }} — values pulled from central constants.* diff --git a/docs/33_333_QUOTA_OVERVIEW.md b/docs/33_333_QUOTA_OVERVIEW.md new file mode 100755 index 00000000..65691938 --- /dev/null +++ b/docs/33_333_QUOTA_OVERVIEW.md @@ -0,0 +1,120 @@ +# Free‑Tier Quota — **{{ quota_anon }}/ {{ quota_token }} Scans per UTC Day** + +Stella Ops is free for individual developers and small teams. +To avoid registry abuse the scanner enforces a **two‑tier daily quota** +— fully offline capable. + +| Mode | Daily ceiling | How to obtain | +|------|---------------|---------------| +| **Anonymous** | **{{ quota_anon }} scans** | No registration. Works online or air‑gapped. | +| **Free JWT token** | **{{ quota_token }} scans** | Email `token@stella-ops.org` (blank body). Bot replies with a signed JWT. | + +*Soft reminder banner appears at 200 scans. Exceeding the limit never blocks – +the CLI/UI introduce a delay, detailed below.* + +--- + +## 1 · Token structure + +```jsonc +{ + "iss": "stella-ops.org", + "sub": "free-tier", + "tid": "7d2285…", // 32‑byte random token‑ID + "tier": {{ quota_token }}, // daily scans allowed + "exp": 1767139199 // POSIX seconds (mandatory) – token expiry +} +```` + +* The **token‑ID (`tid`)** – not the e‑mail – is hashed *(SHA‑256 + salt)* + and stored for counter lookup. +* Verification uses the bundled public key (`keys/cosign.pub`) so **offline + hosts validate tokens locally**. An optional `exp` claim may be present; + if absent, the default is a far‑future timestamp used solely for schema + compatibility. + +--- + +## 2 · Enforcement algorithm (rev 2.1) + +| Step | Operation | Typical latency | +| ---- | ------------------------------------------------------------------------------ | ------------------------------------ | +| 1 | `key = sha256(ip)` *or* `sha256(tid)` | < 0.1 ms | +| 2 | `count = INCR quota:` in Redis (24 h TTL) | 0.2 ms (Lua) | +| 3 | If `count > limit` → `WAIT delay_ms` | first 30 × 5 000 ms → then 60 000 ms | +| 4 | Return HTTP 429 **only if** `delay > 60 s` (should never fire under free tier) | — | + +*Counters reset at **00:00 UTC**.* + +--- + +## 3 · CLI / API integration + +```bash +# Example .env +docker run --rm \ + -e DOCKER_HOST="$DOCKER_HOST" \ # remote‑daemon pointer + -v "$WORKSPACE/${SBOM_FILE}:/${SBOM_FILE}:ro" \ # mount SBOM under same name at container root + -e STELLA_OPS_URL="https://${STELLA_URL}" \ # where the CLI posts findings + "$STELLA_URL/registry/stella-cli:latest" \ + scan --sbom "/${SBOM_FILE}" "$IMAGE" +``` + +*No JWT? → scanner defaults to anonymous quota.* + +--- + +## 4 · Data retention & privacy + +| Data | Retention | Purpose | +| ---------------------- | ------------------------------------ | ---------------- | +| IP hash (`quota:ip:*`) | 7 days, then salted hash only | Abuse rate‑limit | +| Token‑ID hash | Until revoked | Counter lookup | +| E‑mail (token request) | ≤ 7 days unless newsletters opted‑in | Deliver the JWT | + +*No personal data leaves your infrastructure when running offline.* + +--- + +## 5 · Common questions + +
+What happens at exactly 200 scans? + +> The UI/CLI shows a yellow “fair‑use reminder”. +> No throttling is applied yet. +> Once you cross the full limit, the **first 30** over‑quota scans incur a +> 5‑second delay; further excess scans delay **60 s** each. + +
+ +
+Does the quota differ offline? + +> No. Counters are evaluated locally in Redis; the same limits apply even +> without Internet access. + +
+ +
+Can I reset counters manually? + +> Yes – delete the `quota:*` keys in Redis, but we recommend letting them +> expire at midnight to keep statistics meaningful. + +
+ +--- + +## 6 · Revision history + +| Version | Date | Notes | +| ------- | ---------- | ------------------------------------------------------------------- | +| **2.1** | 2025‑07‑16 | Consolidated into single source; delays re‑tuned (30 × 5 s → 60 s). | +|  2.0 | 2025‑04‑07 | Switched counters from Mongo to Redis. | +|  1.0 | 2024‑12‑20 | Initial free‑tier design. | + +--- + +**Authoritative source** — any doc or website section that references quotas +*must* link to this file instead of duplicating text. diff --git a/docs/40_ARCHITECTURE_OVERVIEW.md b/docs/40_ARCHITECTURE_OVERVIEW.md new file mode 100755 index 00000000..15b2c05c --- /dev/null +++ b/docs/40_ARCHITECTURE_OVERVIEW.md @@ -0,0 +1,133 @@ +# Stella Ops — High‑Level Architecture + + + +This document offers a birds‑eye view of how the major components interact, +why the system leans *monolith‑plus‑plug‑ins*, and where extension points live. + +> For a *timeline* of when features arrive, see the public +> [road‑map](/roadmap/) — no version details are repeated here. + +--- + +## 0 · Guiding principles + +| Principle | Rationale | +|-----------|-----------| +| **SBOM‑first** | Scan existing CycloneDX/SPDX if present; fall back to layer unpack. | +| **Δ‑processing** | Re‑analyse only changed layers; reduces P95 warm path to \< 5 s. | +| **All‑managed code** | Entire stack is 100 % managed (.NET / TypeScript); no `unsafe` blocks or native extensions — eases review and reproducible builds. | +| **Restart‑time plug‑ins** | Avoids the attack surface of runtime DLL injection; still allows custom scanners & exporters. | +| **Sovereign‑by‑design** | No mandatory outbound traffic; Offline Kit distributes feeds. | + +--- + +## 1 · Module graph + +```mermaid +graph TD + A(API Gateway) + B1(Scanner Core
.NET latest LTS) + B2(Feedser service\n(vuln ingest/merge/export)) + B3(Policy Engine OPA) + C1(Redis 7) + C2(MongoDB 7) + D(UI SPA
Angular latest version) + A -->|gRPC| B1 + B1 -->|async| B2 + B1 -->|OPA| B3 + B1 --> C1 + B1 --> C2 + A -->|REST/WS| D +```` + +--- + +## 2 · Key components + +| Component | Language / tech | Responsibility | +| ---------------------------- | --------------------- | ---------------------------------------------------- | +| **API Gateway** | ASP.NET Minimal API | Auth (JWT), quotas, request routing | +| **Scanner Core** | C# 12, Polly | Layer diffing, SBOM generation, vuln correlation | +| **Feedser (vulnerability ingest/merge/export service)** | C# source-gen workers | Consolidate NVD + regional CVE feeds into the canonical MongoDB store and drive JSON / Trivy DB exports | +| **Policy Engine** | OPA (Rego) | admission decisions, custom org rules | +| **Redis 7** | Key‑DB compatible | LRU cache, quota counters | +| **MongoDB 7** | WiredTiger | SBOM & findings storage | +| **Angular {{ angular }} UI** | RxJS, Tailwind | Dashboard, reports, admin UX | + +--- + +## 3 · Plug‑in system + +* Discovered once at start‑up from `/opt/stella/plugins/**`. +* Runs under Linux user `stella‑plugin` (UID 1001). +* Extension points: + + * `ISbomMutator` + * `IVulnerabilityProvider` + * `IResultSink` + * Policy files (`*.rego`) +* Each DLL is SHA‑256 hashed; digest embedded in the run report for provenance. + +Hot‑plugging is deferred until after v 1.0 for security review. + +--- + +## 4 · Data & control flow + +1. **Client** calls `/api/scan` with image reference. +2. **Gateway** enforces quota, forwards to **Scanner Core** via gRPC. +3. **Core**: + + * Queries Redis for cached SBOM. + * If miss → pulls layers, generates SBOM. + * Executes plug‑ins (mutators, additional scanners). +4. **Policy Engine** evaluates `scanResult` document. +5. **Findings** stored in MongoDB; WebSocket event notifies UI. +6. **ResultSink plug‑ins** export to Slack, Splunk, JSON file, etc. + +--- + +## 5 · Security hardening + +| Surface | Mitigation | +| ----------------- | ------------------------------------------------------------ | +| Container runtime | Distroless base, non‑root UID, seccomp + AppArmor | +| Plug‑in sandbox | Separate UID, SELinux profile, cgroup 1 CPU / 256 MiB | +| Supply chain | Cosign signatures, in‑toto SLSA Level 3 (target) | +| Secrets | `Docker secrets` or K8s `Secret` mounts; never hard‑coded | +| Quota abuse | Redis rate‑limit gates (see `30_QUOTA_ENFORCEMENT_FLOW1.md`) | + +--- + +## 6 · Build & release pipeline (TL;DR) + +* **Git commits** trigger CI → unit / integration / E2E tests. +* Successful merge to `main`: + + * Build `.NET {{ dotnet }}` trimmed self‑contained binary. + * `docker build --sbom=spdx-json`. + * Sign image and tarball with Cosign. + * Attach SBOM + provenance; push to registry and download portal. + +--- + +## 7 · Future extraction path + +Although the default deployment is a single container, each sub‑service can be +extracted: + +* Feedser → standalone cron pod. +* Policy Engine → side‑car (OPA) with gRPC contract. +* ResultSink → queue worker (RabbitMQ or Azure Service Bus). + +Interfaces are stable **as of v0.2 β**; extraction requires a recompilation +only, not a fork of the core. + +--- + +*Last updated {{ "now" | date: "%Y‑%m‑%d" }} – constants auto‑injected.* diff --git a/docs/60_POLICY_TEMPLATES.md b/docs/60_POLICY_TEMPLATES.md new file mode 100755 index 00000000..08eaca8d --- /dev/null +++ b/docs/60_POLICY_TEMPLATES.md @@ -0,0 +1,101 @@ +# Policy Templates — YAML & Rego Examples + +Stella Ops lets you enforce *pass / fail* rules in two ways: + +1. **YAML “quick policies”** — simple equality / inequality checks. +2. **OPA Rego modules** — full‑power logic for complex organisations. + +> **Precedence:** If the same image is subject to both a YAML rule *and* a Rego +> module, the **Rego result wins**. That is, `deny` in Rego overrides any +> `allow` in YAML. + +--- + +## 1 · YAML quick policy + +```yaml +# file: policies/root_user.yaml +version: 1 +id: root-user +description: Disallow images that run as root +severity: high + +rules: + - field: ".config.user" + operator: "equals" + value: "root" + deny_message: "Image runs as root — block." +```` + +Place the file under `/opt/stella/plugins/policies/`. + +--- + +## 2 · Rego example (deny on critical CVE) + +```rego +# file: policies/deny_critical.rego +package stella.policy + +default deny = [] + +deny[msg] { + some f + input.findings[f].severity == "critical" + msg := sprintf("Critical CVE %s – build blocked", [input.findings[f].id]) +} +``` + +*Input schema* — the Rego `input` document matches the public +`ScanResult` POCO (see SDK). Use the bundled JSON schema in +`share/schemas/scanresult.schema.json` for IDE autocompletion. + +--- + +## 3 · Pass‑through warnings (Rego) + +Return a `warn` array to surface non‑blocking messages in the UI: + +```rego +package stella.policy + +warn[msg] { + input.image.base == "ubuntu:16.04" + msg := "Image uses EOL Ubuntu 16.04 — please upgrade." +} +``` + +Warnings decrement the **quality score** but do *not* affect the CLI exit +code. + +--- + +## 4 · Testing policies locally + +```bash +# run policy evaluation without pushing to DB +stella scan alpine:3.20 --policy-only \ + --policies ./policies/ +``` + +The CLI prints `PASS`, `WARN` or `DENY` plus structured JSON. + +Unit‑test your Rego modules with the OPA binary: + +```bash +opa test policies/ +``` + +--- + +## 5 · Developer quick‑start (plug‑ins) + +Need logic beyond Rego? Implement a plug‑in via **C#/.NET {{ dotnet }}** and +the `StellaOps.SDK` NuGet: + +* Tutorial: [`dev/30_PLUGIN_DEV_GUIDE.md`](dev/30_PLUGIN_DEV_GUIDE.md) +* Quick reference: `/plugins/` + +--- + +*Last updated {{ "now" | date: "%Y‑%m‑%d" }} — constants auto‑injected.* diff --git a/docs/ARCHITECTURE_FEEDSER.md b/docs/ARCHITECTURE_FEEDSER.md new file mode 100644 index 00000000..f1df1515 --- /dev/null +++ b/docs/ARCHITECTURE_FEEDSER.md @@ -0,0 +1,190 @@ +# ARCHITECTURE.md — **StellaOps.Feedser** + +> **Goal**: Build a sovereign-ready, self-hostable **feed-merge service** that ingests authoritative vulnerability sources, normalizes and de-duplicates them into **MongoDB**, and exports **JSON** and **Trivy-compatible DB** artifacts. +> **Form factor**: Long-running **Web Service** with **REST APIs** (health, status, control) and an embedded **internal cron scheduler**. Controllable by StellaOps.Cli (# stella db ...) +> **No signing inside Feedser** (signing is a separate pipeline step). +> **Runtime SDK baseline**: .NET 10 Preview 7 (SDK 10.0.100-preview.7.25380.108) targeting `net10.0`, aligned with the deployed api.stella-ops.org service. +> **Four explicit stages**: +> +> 1. **Source Download** → raw documents. +> 2. **Parse & Normalize** → schema-validated DTOs enriched with canonical identifiers. +> 3. **Merge & Deduplicate** → precedence-aware canonical records persisted to MongoDB. +> 4. **Export** → JSON or TrivyDB (full or delta), then (externally) sign/publish. + +--- + +## 1) Naming & Solution Layout + +**Source connectors** namespace prefix: `StellaOps.Feedser.Source.*` +**Exporters**: + +* `StellaOps.Feedser.Exporter.Json` +* `StellaOps.Feedser.Exporter.TrivyDb` + +**Projects** (`/src`): + +``` +StellaOps.Feedser.WebService/ # ASP.NET Core (Minimal API, net10.0 preview) WebService + embedded scheduler +StellaOps.Feedser.Core/ # Domain models, pipelines, merge/dedupe engine, jobs orchestration +StellaOps.Feedser.Models/ # Canonical POCOs, JSON Schemas, enums +StellaOps.Feedser.Storage.Mongo/ # Mongo repositories, GridFS access, indexes, resume "flags" +StellaOps.Feedser.Source.Common/ # HTTP clients, rate-limiters, schema validators, parsers utils +StellaOps.Feedser.Source.Cve/ +StellaOps.Feedser.Source.Nvd/ +StellaOps.Feedser.Source.Ghsa/ +StellaOps.Feedser.Source.Osv/ +StellaOps.Feedser.Source.Jvn/ +StellaOps.Feedser.Source.CertCc/ +StellaOps.Feedser.Source.Kev/ +StellaOps.Feedser.Source.Kisa/ +StellaOps.Feedser.Source.CertIn/ +StellaOps.Feedser.Source.CertFr/ +StellaOps.Feedser.Source.CertBund/ +StellaOps.Feedser.Source.Acsc/ +StellaOps.Feedser.Source.Cccs/ +StellaOps.Feedser.Source.Ru.Bdu/ # HTML→schema with LLM fallback (gated) +StellaOps.Feedser.Source.Ru.Nkcki/ # PDF/HTML bulletins → structured +StellaOps.Feedser.Source.Vndr.Msrc/ +StellaOps.Feedser.Source.Vndr.Cisco/ +StellaOps.Feedser.Source.Vndr.Oracle/ +StellaOps.Feedser.Source.Vndr.Adobe/ +StellaOps.Feedser.Source.Vndr.Apple/ +StellaOps.Feedser.Source.Vndr.Chromium/ +StellaOps.Feedser.Source.Vndr.Vmware/ +StellaOps.Feedser.Source.Distro.RedHat/ +StellaOps.Feedser.Source.Distro.Ubuntu/ +StellaOps.Feedser.Source.Distro.Debian/ +StellaOps.Feedser.Source.Distro.Suse/ +StellaOps.Feedser.Source.Ics.Cisa/ +StellaOps.Feedser.Source.Ics.Kaspersky/ +StellaOps.Feedser.Normalization/ # Canonical mappers, validators, version-range normalization +StellaOps.Feedser.Merge/ # Identity graph, precedence, deterministic merge +StellaOps.Feedser.Exporter.Json/ +StellaOps.Feedser.Exporter.TrivyDb/ +StellaOps.Feedser..Tests/ # Component-scoped unit/integration suites (Core, Storage.Mongo, Source.*, Exporter.*, WebService, etc.) +``` + +--- + +## 2) Runtime Shape + +**Process**: single service (`StellaOps.Feedser.WebService`) + +* `Program.cs`: top-level entry using **Generic Host**, **DI**, **Options** binding from `appsettings.json` + environment + optional `feedser.yaml`. +* Built-in **scheduler** (cron-like) + **job manager** with **distributed locks** in Mongo to prevent overlaps, enforce timeouts, allow cancel/kill. +* **REST APIs** for health/readiness/progress/trigger/kill/status. + +**Key NuGet concepts** (indicative): `MongoDB.Driver`, `Polly` (retry/backoff), `System.Threading.Channels`, `Microsoft.Extensions.Http`, `Microsoft.Extensions.Hosting`, `Serilog`, `OpenTelemetry`. + +--- + +## 3) Data Storage — **MongoDB** (single source of truth) + +**Database**: `feedser` +**Write concern**: `majority` for merge/export state, `acknowledged` for raw docs. +**Collections** (with “flags”/resume points): + +* `source` + * `_id`, `name`, `type`, `baseUrl`, `auth`, `notes`. +* `source_state` + * Keys: `sourceName` (unique), `enabled`, `cursor`, `lastSuccess`, `failCount`, `backoffUntil`, `paceOverrides`, `paused`. + * Drives incremental fetch/parse/map resume and operator pause/pace controls. +* `document` + * `_id`, `sourceName`, `uri`, `fetchedAt`, `sha256`, `contentType`, `status`, `metadata`, `gridFsId`, `etag`, `lastModified`. + * Index `{sourceName:1, uri:1}` unique; optional TTL for superseded versions. +* `dto` + * `_id`, `sourceName`, `documentId`, `schemaVer`, `payload` (BSON), `validatedAt`. + * Index `{sourceName:1, documentId:1}`. +* `advisory` + * `_id`, `advisoryKey`, `title`, `summary`, `lang`, `published`, `modified`, `severity`, `exploitKnown`. + * Unique `{advisoryKey:1}` plus indexes on `modified` and `published`. +* `alias` + * `advisoryId`, `scheme`, `value` with index `{scheme:1, value:1}`. +* `affected` + * `advisoryId`, `platform`, `name`, `versionRange`, `cpe`, `purl`, `fixedBy`, `introducedVersion`. + * Index `{platform:1, name:1}`, `{advisoryId:1}`. +* `reference` + * `advisoryId`, `url`, `kind`, `sourceTag` (e.g., advisory/patch/kb). +* Flags collections: `kev_flag`, `ru_flags`, `jp_flags`, `psirt_flags` keyed by `advisoryId`. +* `merge_event` + * `_id`, `advisoryKey`, `beforeHash`, `afterHash`, `mergedAt`, `inputs` (document ids). +* `export_state` + * `_id` (`json`/`trivydb`), `baseExportId`, `baseDigest`, `lastFullDigest`, `lastDeltaDigest`, `exportCursor`, `targetRepo`, `exporterVersion`. +* `locks` + * `_id` (`jobKey`), `holder`, `acquiredAt`, `heartbeatAt`, `leaseMs`, `ttlAt` (TTL index cleans dead locks). +* `jobs` + * `_id`, `type`, `args`, `state`, `startedAt`, `endedAt`, `error`, `owner`, `heartbeatAt`, `timeoutMs`. + +**GridFS buckets**: `fs.documents` for raw large payloads; referenced by `document.gridFsId`. + +--- + +## 4) Job & Scheduler Model + +* Scheduler stores cron expressions per source/exporter in config; persists next-run pointers in Mongo. +* Jobs acquire locks (`locks` collection) to ensure singleton execution per source/exporter. +* Supports manual triggers via API endpoints (`POST /jobs/{type}`) and pause/resume toggles per source. + +--- + +## 5) Connector Contracts + +Connectors implement: + +```csharp +public interface IFeedConnector { + string SourceName { get; } + Task FetchAsync(IServiceProvider sp, CancellationToken ct); + Task ParseAsync(IServiceProvider sp, CancellationToken ct); + Task MapAsync(IServiceProvider sp, CancellationToken ct); +} +``` + +* Fetch populates `document` rows respecting rate limits, conditional GET, and `source_state.cursor`. +* Parse validates schema (JSON Schema, XSD) and writes sanitized DTO payloads. +* Map produces canonical advisory rows + provenance entries; must be idempotent. +* Base helpers in `StellaOps.Feedser.Source.Common` provide HTTP clients, retry policies, and watermark utilities. + +--- + +## 6) Merge & Normalization + +* Canonical model stored in `StellaOps.Feedser.Models` with serialization contracts used by storage/export layers. +* `StellaOps.Feedser.Normalization` handles NEVRA/EVR/PURL range parsing, CVSS normalization, localization. +* `StellaOps.Feedser.Merge` builds alias graphs keyed by CVE first, then falls back to vendor/regional IDs. +* Precedence rules: PSIRT/OVAL overrides generic ranges; KEV only toggles exploitation; regional feeds enrich severity but don’t override vendor truth. +* Determinism enforced via canonical JSON hashing logged in `merge_event`. + +--- + +## 7) Exporters + +* JSON exporter mirrors `aquasecurity/vuln-list` layout with deterministic ordering and reproducible timestamps. +* Trivy DB exporter initially shells out to `trivy-db` builder; later will emit BoltDB directly. +* `StellaOps.Feedser.Storage.Mongo` provides cursors for delta exports based on `export_state.exportCursor`. +* Export jobs produce OCI tarballs (layer media type `application/vnd.aquasec.trivy.db.layer.v1.tar+gzip`) and optionally push via ORAS. + +--- + +## 8) Observability + +* Serilog structured logging with enrichment fields (`source`, `uri`, `stage`, `durationMs`). +* OpenTelemetry traces around fetch/parse/map/export; metrics for rate limit hits, schema failures, dedupe ratios, package size. +* Prometheus scraping endpoint served by WebService. + +--- + +## 9) Security Considerations + +* Offline-first: connectors only reach allowlisted hosts. +* BDU LLM fallback gated by config flag; logs audit trail with confidence score. +* No secrets written to logs; secrets loaded via environment or mounted files. +* Signing handled outside Feedser pipeline. + +--- + +## 10) Deployment Notes + +* Default storage MongoDB; for air-gapped, bundle Mongo image + seeded data backup. +* Horizontal scale achieved via multiple web service instances sharing Mongo locks. +* Provide `feedser.yaml` template describing sources, rate limits, and export settings. diff --git a/docs/README.md b/docs/README.md new file mode 100755 index 00000000..c1a12ece --- /dev/null +++ b/docs/README.md @@ -0,0 +1,67 @@ +# Stella Ops + +> **Self‑hosted, SBOM‑first DevSecOps platform – offline‑friendly, AGPL‑3.0, free up to {{ quota_token }} scans per UTC day (soft delay only, never blocks).** + +Stella Ops lets you discover container vulnerabilities in **< 5 s** without sending a single byte outside your network. +Everything here is open‑source and versioned — when you check out a git tag, the docs match the code you are running. + +--- + +## 🚀 Start here (first 60 minutes) + +| Step | What you will learn | Doc | +|------|--------------------|-----| +| 1 ️⃣ | 90‑second elevator pitch & pillars | **[What Is Stella Ops?](01_WHAT_IS_IT.md)** | +| 2 ️⃣ | Pain points it solves | **[Why Does It Exist?](02_WHY.md)** | +| 3 ️⃣ | Install & run a scan in 10 min | **[Install Guide](21_INSTALL_GUIDE.md)** | +| 4 ️⃣ | Components & data‑flow | **[High‑Level Architecture](07_HIGH_LEVEL_ARCHITECTURE.md)** | +| 5 ️⃣ | Integrate the CLI / REST API | **[API & CLI Reference](09_API_CLI_REFERENCE.md)** | +| 6 ️⃣ | Vocabulary used throughout the docs | **[Glossary](14_GLOSSARY_OF_TERMS.md)** | + +--- + +## 📚 Complete Table of Contents + +
+Click to expand the full docs index + +### Overview +- **01 – [What Is Stella Ops?](01_WHAT_IS_IT.md)** +- **02 – [Why Does It Exist?](02_WHY.md)** +- **03 – [Vision & Road‑map](03_VISION.md)** +- **04 – [Feature Matrix](04_FEATURE_MATRIX.md)** + +### Reference & concepts +- **05 – [System Requirements Specification](05_SYSTEM_REQUIREMENTS_SPEC.md)** +- **07 – [High‑Level Architecture](40_ARCHITECTURE_OVERVIEW.md)** +- **08 – Module Specifications** + - [README](08_MODULE_SPECIFICATIONS/README.md) + - [`backend_api.md`](08_MODULE_SPECIFICATIONS/backend_api.md) + - [`zastava_scanner.md`](08_MODULE_SPECIFICATIONS/zastava_scanner.md) + - [`registry_scanner.md`](08_MODULE_SPECIFICATIONS/registry_scanner.md) + - [`nightly_scheduler.md`](08_MODULE_SPECIFICATIONS/nightly_scheduler.md) +- **09 – [API & CLI Reference](09_API_CLI_REFERENCE.md)** +- **10 – [Plug‑in SDK Guide](10_PLUGIN_SDK_GUIDE.md)** +- **11 – [Data Schemas](11_DATA_SCHEMAS.md)** +- **12 – [Performance Workbook](12_PERFORMANCE_WORKBOOK.md)** +- **13 – [Release‑Engineering Playbook](13_RELEASE_ENGINEERING_PLAYBOOK.md)** + +### User & operator guides +- **14 – [Glossary](14_GLOSSARY_OF_TERMS.md)** +- **15 – [UI Guide](15_UI_GUIDE.md)** +- **17 – [Security Hardening Guide](17_SECURITY_HARDENING_GUIDE.md)** +- **18 – [Coding Standards](18_CODING_STANDARDS.md)** +- **19 – [Test‑Suite Overview](19_TEST_SUITE_OVERVIEW.md)** +- **21 – [Install Guide](21_INSTALL_GUIDE.md)** +- **22 – [CI/CD Recipes Library](ci/20_CI_RECIPES.md)** +- **23 – [FAQ](23_FAQ_MATRIX.md)** +- **24 – [Offline Update Kit Admin Guide](24_OUK_ADMIN_GUIDE.md)** + +### Legal & licence +- **29 – [Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)** + +
+ +--- + +© 2025 Stella Ops contributors – licensed AGPL‑3.0‑or‑later diff --git a/docs/_includes/CONSTANTS.md b/docs/_includes/CONSTANTS.md new file mode 100755 index 00000000..efde601b --- /dev/null +++ b/docs/_includes/CONSTANTS.md @@ -0,0 +1,18 @@ +### `docs/_includes/CONSTANTS.md` + +```yaml +--- +# ───────────────────────────────────────────────────────────────────────────── +# Shared constants for both the technical docs (Markdown) and the marketing +# site (Nunjucks). Eleventy injects these variables into every template. +# Never hard‑code the values elsewhere — lint‑ci will block the merge. +# ───────────────────────────────────────────────────────────────────────────── + +dotnet: "10 LTS" # Runs on .NET 10 (LTS channel) +angular: "20" # Front‑end framework major +quota_anon: 33 # Anonymous daily scans +quota_token: 333 # Daily scans with free JWT +slowdown: "5–60 s" # Delay window after exceeding quota + +# Add new keys here; update the docs linter pattern in .gitlab-ci.yml. +--- \ No newline at end of file diff --git a/docs/ci/20_CI_RECIPES.md b/docs/ci/20_CI_RECIPES.md new file mode 100755 index 00000000..4ad86464 --- /dev/null +++ b/docs/ci/20_CI_RECIPES.md @@ -0,0 +1,258 @@ +# Stella Ops CI Recipes — (2025‑08‑04) + +## 0 · Key variables (export these once) + +| Variable | Meaning | Typical value | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | +| `STELLA_URL` | Host that: ① stores the **CLI** & **SBOM‑builder** images under `/registry` **and** ② receives API calls at `https://$STELLA_URL` | `stella-ops.ci.acme.example` | +| `DOCKER_HOST` | How containers reach your Docker daemon (because we no longer mount `/var/run/docker.sock`) | `tcp://docker:2375` | +| `WORKSPACE` | Directory where the pipeline stores artefacts (SBOM file) | `$(pwd)` | +| `IMAGE` | The image you are building & scanning | `acme/backend:sha-${COMMIT_SHA}` | +| `SBOM_FILE` | Immutable SBOM name – `‑YYYYMMDDThhmmssZ.sbom.json` | `acme_backend_sha‑abc123‑20250804T153050Z.sbom.json` | + +```bash +export STELLA_URL="stella-ops.ci.acme.example" +export DOCKER_HOST="tcp://docker:2375" # Jenkins/Circle often expose it like this +export WORKSPACE="$(pwd)" +export IMAGE="acme/backend:sha-${COMMIT_SHA}" +export SBOM_FILE="$(echo "${IMAGE}" | tr '/:+' '__')-$(date -u +%Y%m%dT%H%M%SZ).sbom.json" +``` + +--- + +## 1 · SBOM creation strategies + +### Option A – **Buildx attested SBOM** (preferred if you can use BuildKit) + +You pass **two build args** so the Dockerfile can run the builder and copy the result out of the build context. + +```bash +docker buildx build \ + --build-arg STELLA_SBOM_BUILDER="$STELLA_URL/registry/stella-sbom-builder:latest" \ + --provenance=true --sbom=true \ + --build-arg SBOM_FILE="$SBOM_FILE" \ + -t "$IMAGE" . +``` + +**If you **cannot** use Buildx, use Option B below.** The older “run a builder stage inside the Dockerfile” pattern is unreliable for producing an SBOM of the final image. + +```Dockerfile + +ARG STELLA_SBOM_BUILDER +ARG SBOM_FILE + +FROM $STELLA_SBOM_BUILDER as sbom +ARG IMAGE +ARG SBOM_FILE +RUN $STELLA_SBOM_BUILDER build --image $IMAGE --output /out/$SBOM_FILE + +# ---- actual build stages … ---- +FROM alpine:3.20 +COPY --from=sbom /out/$SBOM_FILE / # (optional) keep or discard + +# (rest of your Dockerfile) +``` + +### Option B – **External builder step** (works everywhere; recommended baseline if Buildx isn’t available) + +*(keep this block if your pipeline already has an image‑build step that you can’t modify)* + +```bash +docker run --rm \ + -e DOCKER_HOST="$DOCKER_HOST" \ # let builder reach the daemon remotely + -v "$WORKSPACE:/workspace" \ # place SBOM beside the source code + "$STELLA_URL/registry/stella-sbom-builder:latest" \ + build --image "$IMAGE" --output "/workspace/${SBOM_FILE}" +``` + +--- + +## 2 · Scan the image & upload results + +```bash +docker run --rm \ + -e DOCKER_HOST="$DOCKER_HOST" \ # remote‑daemon pointer + -v "$WORKSPACE/${SBOM_FILE}:/${SBOM_FILE}:ro" \ # mount SBOM under same name at container root + -e STELLA_OPS_URL="https://${STELLA_URL}" \ # where the CLI posts findings + "$STELLA_URL/registry/stella-cli:latest" \ + scan --sbom "/${SBOM_FILE}" "$IMAGE" +``` + +The CLI returns **exit 0** if policies pass, **>0** if blocked — perfect for failing the job. + +--- + +## 3 · CI templates + +Below are minimal, cut‑and‑paste snippets. +**Feel free to delete Option B** if you adopt Option A. + +### 3.1 Jenkins (Declarative Pipeline) + +```groovy +pipeline { + agent { docker { image 'docker:25' args '--privileged' } } // gives us /usr/bin/docker + environment { + STELLA_URL = 'stella-ops.ci.acme.example' + DOCKER_HOST = 'tcp://docker:2375' + IMAGE = "acme/backend:${env.BUILD_NUMBER}" + SBOM_FILE = "acme_backend_${env.BUILD_NUMBER}-${new Date().format('yyyyMMdd\'T\'HHmmss\'Z\'', TimeZone.getTimeZone('UTC'))}.sbom.json" + } + stages { + stage('Build image + SBOM (Option A)') { + steps { + sh ''' + docker build \ + --build-arg STELLA_SBOM_BUILDER="$STELLA_URL/registry/stella-sbom-builder:latest" \ + --build-arg SBOM_FILE="$SBOM_FILE" \ + -t "$IMAGE" . + ''' + } + } + /* ---------- Option B fallback (when you must keep the existing build step as‑is) ---------- + stage('SBOM builder (Option B)') { + steps { + sh ''' + docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + -v "$WORKSPACE:/workspace" \ + "$STELLA_URL/registry/stella-sbom-builder:latest" \ + build --image "$IMAGE" --output "/workspace/${SBOM_FILE}" + ''' + } + } + ------------------------------------------------------------------------------------------ */ + stage('Scan & upload') { + steps { + sh ''' + docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + -v "$WORKSPACE/${SBOM_FILE}:/${SBOM_FILE}:ro" \ + -e STELLA_OPS_URL="https://$STELLA_URL" \ + "$STELLA_URL/registry/stella-cli:latest" \ + scan --sbom "/${SBOM_FILE}" "$IMAGE" + ''' + } + } + } +} +``` + +--- + +### 3.2 CircleCI `.circleci/config.yml` + +```yaml +version: 2.1 +jobs: + stella_scan: + docker: + - image: cimg/base:stable # baremetal image with Docker CLI + environment: + STELLA_URL: stella-ops.ci.acme.example + DOCKER_HOST: tcp://docker:2375 # Circle’s “remote Docker” socket + steps: + - checkout + + - run: + name: Compute vars + command: | + echo 'export IMAGE="acme/backend:${CIRCLE_SHA1}"' >> $BASH_ENV + echo 'export SBOM_FILE="$(echo acme/backend:${CIRCLE_SHA1} | tr "/:+" "__")-$(date -u +%Y%m%dT%H%M%SZ).sbom.json"' >> $BASH_ENV + - run: + name: Build image + SBOM (Option A) + command: | + docker build \ + --build-arg STELLA_SBOM_BUILDER="$STELLA_URL/registry/stella-sbom-builder:latest" \ + --build-arg SBOM_FILE="$SBOM_FILE" \ + -t "$IMAGE" . + # --- Option B fallback (when you must keep the existing build step as‑is) --- + #- run: + # name: SBOM builder (Option B) + # command: | + # docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + # -v "$PWD:/workspace" \ + # "$STELLA_URL/registry/stella-sbom-builder:latest" \ + # build --image "$IMAGE" --output "/workspace/${SBOM_FILE}" + - run: + name: Scan + command: | + docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + -v "$PWD/${SBOM_FILE}:/${SBOM_FILE}:ro" \ + -e STELLA_OPS_URL="https://$STELLA_URL" \ + "$STELLA_URL/registry/stella-cli:latest" \ + scan --sbom "/${SBOM_FILE}" "$IMAGE" +workflows: + stella: + jobs: [stella_scan] +``` + +--- + +### 3.3 Gitea Actions `.gitea/workflows/stella.yml` + +*(Gitea 1.22+ ships native Actions compatible with GitHub syntax)* + +```yaml +name: Stella Scan +on: [push] + +jobs: + stella: + runs-on: ubuntu-latest + env: + STELLA_URL: ${{ secrets.STELLA_URL }} + DOCKER_HOST: tcp://docker:2375 # provided by the docker:dind service + services: + docker: + image: docker:dind + options: >- + --privileged + steps: + - uses: actions/checkout@v4 + + - name: Compute vars + id: vars + run: | + echo "IMAGE=ghcr.io/${{ gitea.repository }}:${{ gitea.sha }}" >> $GITEA_OUTPUT + echo "SBOM_FILE=$(echo ghcr.io/${{ gitea.repository }}:${{ gitea.sha }} | tr '/:+' '__')-$(date -u +%Y%m%dT%H%M%SZ).sbom.json" >> $GITEA_OUTPUT + + - name: Build image + SBOM (Option A) + run: | + docker build \ + --build-arg STELLA_SBOM_BUILDER="${STELLA_URL}/registry/stella-sbom-builder:latest" \ + --build-arg SBOM_FILE="${{ steps.vars.outputs.SBOM_FILE }}" \ + -t "${{ steps.vars.outputs.IMAGE }}" . + + # --- Option B fallback (when you must keep the existing build step as‑is) --- + #- name: SBOM builder (Option B) + # run: | + # docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + # -v "$(pwd):/workspace" \ + # "${STELLA_URL}/registry/stella-sbom-builder:latest" \ + # build --image "${{ steps.vars.outputs.IMAGE }}" --output "/workspace/${{ steps.vars.outputs.SBOM_FILE }}" + + - name: Scan + run: | + docker run --rm -e DOCKER_HOST="$DOCKER_HOST" \ + -v "$(pwd)/${{ steps.vars.outputs.SBOM_FILE }}:/${{ steps.vars.outputs.SBOM_FILE }}:ro" \ + -e STELLA_OPS_URL="https://${STELLA_URL}" \ + "${STELLA_URL}/registry/stella-cli:latest" \ + scan --sbom "/${{ steps.vars.outputs.SBOM_FILE }}" "${{ steps.vars.outputs.IMAGE }}" +``` + +--- + +## 4 · Troubleshooting cheat‑sheet + +| Symptom | Root cause | First things to try | +| ------------------------------------- | --------------------------- | --------------------------------------------------------------- | +| `no such host $STELLA_URL` | DNS typo or VPN outage | `ping $STELLA_URL` from runner | +| `connection refused` when CLI uploads | Port 443 blocked | open firewall / check ingress | +| `failed to stat /.json` | SBOM wasn’t produced | Did Option A actually run builder? If not, enable Option B | +| `registry unauthorized` | Runner lacks registry creds | `docker login $STELLA_URL/registry` (store creds in CI secrets) | +| Non‑zero scan exit | Blocking vuln/licence | Open project in Ops UI → triage or waive | + +--- + +### Change log + +* **2025‑08‑04** – Variable clean‑up, removed Docker‑socket & cache mounts, added Jenkins / CircleCI / Gitea examples, clarified Option B comment. diff --git a/docs/cli/20_REFERENCE.md b/docs/cli/20_REFERENCE.md new file mode 100755 index 00000000..941aa692 --- /dev/null +++ b/docs/cli/20_REFERENCE.md @@ -0,0 +1,8 @@ +# CLI Reference (`stella --help`) + +> **Auto‑generated file — do not edit manually.** +> On every tagged release the CI pipeline runs +> `stella --help --markdown > docs/cli/20_REFERENCE.md` +> ensuring this document always matches the shipped binary. + +*(The reference will appear after the first public α release.)* diff --git a/docs/dev/30_PLUGIN_DEV_GUIDE.md b/docs/dev/30_PLUGIN_DEV_GUIDE.md new file mode 100755 index 00000000..12e21827 --- /dev/null +++ b/docs/dev/30_PLUGIN_DEV_GUIDE.md @@ -0,0 +1,146 @@ +# Writing Plug‑ins for Stella Ops SDK *Preview 3* + +> **SDK status:** *Preview 3* is compatible with the **v0.1 α** runtime. +> Interfaces freeze at **v0.2 β**; binary‑breaking changes are still possible +> until then. + +| SDK NuGet | Runtime compat | Notes | +|-----------|---------------|-------| +| `StellaOps.SDK 0.2.0-preview3` | `stella-ops >= 0.1.0-alpha` | Current preview | +| `StellaOps.SDK 0.2.x‑beta` | v0.2 β (Q1 2026) | Interface **freeze** | +| `StellaOps.SDK 1.0.0` | v1.0 GA (Q4 2026) | Semantic Ver from here | + +--- + +## 0 · Extension points + +| Area | Interface / format | Example | +|------|--------------------|---------| +| SBOM mutator | `ISbomMutator` | Inject SPDX licences | +| Additional scanner | `IVulnerabilityProvider` | Rust Crates ecosystem | +| Policy engine | **OPA Rego** file | Custom pass/fail rule | +| Result exporter | `IResultSink` | Slack webhook notifier | + +*Hot‑plugging (live reload) is **post‑1.0**; modules are discovered once +during service start‑up.* + +--- + +## 1 · Five‑minute quick‑start (C# /.NET {{ dotnet }}) + +```bash +dotnet new classlib -n SlackExporter +cd SlackExporter +dotnet add package StellaOps.SDK --version 0.2.0-preview3 +```` + +```csharp +using System.Net.Http.Json; +using StellaOps.Plugin; + +public sealed class SlackSink : IResultSink +{ + private readonly string _webhook = + Environment.GetEnvironmentVariable("SLACK_WEBHOOK") + ?? throw new InvalidOperationException("Missing SLACK_WEBHOOK"); + + public string Name => "Slack Notifier"; + + public async Task ExportAsync(ScanResult result, CancellationToken ct) + { + var payload = new + { + text = $":rotating_light: *{result.Image}* " + + $"→ {result.Findings.Count} findings (max {result.MaxSeverity})" + }; + + using var client = new HttpClient(); + await client.PostAsJsonAsync(_webhook, payload, ct); + } +} +``` + +```bash +dotnet publish -c Release -o out +sudo mkdir -p /opt/stella/plugins/Slack +sudo cp out/SlackExporter.dll /opt/stella/plugins/Slack/ +sudo systemctl restart stella-ops +``` + +Start‑up log: + +``` +[PluginLoader] Loaded 1 plug‑in: + • Slack Notifier +``` + +--- + +## 2 · Packaging rules + +| Item | Rule | +| ------ | ----------------------------------------- | +| Folder | `/opt/stella/plugins//` | +| DLLs | Your plug‑in + non‑GAC deps | +| Config | Env‑vars or `settings.yaml` | +| SBOM | Optional `addon.spdx.json` for provenance | + +--- + +## 3 · Security sandbox + +* Runs as Linux user **`stella‑plugin` (UID 1001)**. +* SELinux/AppArmor profile blocks inbound traffic; outbound :80/443 only. +* cgroup default: **1 CPU / 256 MiB** (adjustable). +* SHA‑256 of every DLL is embedded in the run report. + +--- + +## 4 · Debugging + +| Technique | Command | +| ----------------- | ---------------------------------- | +| Verbose core log | `STELLA_LOG=debug` | +| Per‑plug‑in log | Inject `ILogger` | +| Dry‑run (no fail) | `--plugin-mode warn` | +| Hot reload | *Not supported* (planned post‑1.0) | + +Logs: `/var/log/stella-ops/plugins/YYYY‑MM‑DD.log`. + +--- + +## 5 · Interface reference (Preview 3) + +```csharp +namespace StellaOps.Plugin +{ + public interface ISbomMutator + { + string Name { get; } + Task MutateAsync( + SoftwareBillOfMaterials sbom, + CancellationToken ct = default); + } + + public interface IVulnerabilityProvider + { + string Ecosystem { get; } + Task> QueryAsync( + PackageReference p, CancellationToken ct = default); + } + + public interface IResultSink + { + string Name { get; } + Task ExportAsync( + ScanResult result, CancellationToken ct = default); + } +} +``` + +Full POCO docs: [https://git.stella-ops.org/stella-ops/sdk/-/tree/main/docs/api](https://git.stella-ops.org/stella-ops/sdk/-/tree/main/docs/api). + +--- + +*Last updated {{ "now" | date: "%Y‑%m‑%d" }} – constants auto‑injected.* + diff --git a/docs/license-jwt-quota.md b/docs/license-jwt-quota.md new file mode 100755 index 00000000..f54ee250 --- /dev/null +++ b/docs/license-jwt-quota.md @@ -0,0 +1,123 @@ +--- +title: Offline JWT licence & daily‑run quota +description: How Stella‑Ops enforces a **runs‑per‑day** limit in fully air‑gapped deployments. +nav: + order: 36 +--- + +# JWT‑based daily‑run licence (offline‑capable) + +When *Stella‑Ops* scanners operate entirely **offline**, they cannot phone home +for metering. +Instead, the backend accepts a **signed JSON Web Token (JWT)** that states the +**maximum number of scans per UTC day**. +If no token is supplied, a _grace quota_ of **33 runs/24 h** applies. + +--- + +## 1  Token contents + +| Claim | Purpose | Example | +|-------|---------|---------| +| `sub` | Customer / licensee identifier | `"f47ac10b…"` | +| `iat` | Issued‑at timestamp | `1722566400` | +| `exp` | Absolute licence expiry | `2025‑12‑31T23:59:59Z` | +| `tier` | **Max scans per UTC day** | `{{ quota_token }}` | +| `tid` | Token identifier (32‑byte) | `"7d2285..."` | +| `pkg` | Product SKU / edition | `"stella‑core"` | + +Tokens are signed with **RS256** and verified locally using the bundled public key. +Only the public key ships inside the container; the private key never leaves +the build pipeline. + +--- + +## 2  Obtaining a token + +1. **Request** → `POST /​register { email:"alice@example.org" }` +2. Service hashes the e‑mail (SHA‑256), stores it, and issues a JWT (60 days by default). +3. Token is e‑mailed to you. + +A new request for the same e‑mail returns the **same** token until it nears +expiry, avoiding quota “top‑ups” by re‑registration. + +--- + +## 3  Supplying the token to an air‑gapped stack + +```bash +# recommended +docker run \ + -v /opt/stella/license/alice.jwt:/run/secrets/stella_license.jwt:ro \ + stella‑ops +```` + +Other supported paths: + +| Method | Mount point | Hot‑reload | +| ------------- | ------------------------ | ----------- | +| Docker secret | `/run/secrets/…` | ✓ (inotify) | +| Bind‑mounted | user‑chosen path (above) | ✓ | +| Env variable | `STELLA_LICENSE_JWT` | ✗ restart | + +--- + +## 4  Quota‑enforcement algorithm + +```mermaid +flowchart TD + Start --> Verify[Verify JWT signature] + Verify -->|Invalid| Deny1[Run in non licensed mode] + Verify --> Load[load today's counter UTC] + Load -->|SUM of last 24h scans < daily_quota| Permit[allow scan, add scan] + Permit --> End + Load -->|SUM of last 24h scans ≥ daily_quota| Deny1 +``` + + +## 5  Renewal procedure + +| Scenario | Action | +| -------------- | --------------------------------------------------------------------------------- | +| More capacity | Request new token with higher `daily_quota`; replace file – **no restart needed** | +| Licence expiry | Same as above; new `exp` date | +| Key rotation | Container image ships new public key(s); older tokens still verify | + +--- + +## 6  Fallback limits + +| Situation | Daily quota | +| ----------------------- | ----------------------------------- | +| Valid JWT present | value of `daily_quota` claim ({{ quota_token }}) | +| No JWT | **33** | +| JWT expired (if used) | treated as **anonymous** unless policy enforces hard‑fail | +| Token signature invalid | **0** (reject) | + +--- + +## 7  Threat‑model highlights (future work / optional hardening) + +| Threat | Mitigation | +| --------------------------- | ---------------------------------------------------------------------- | +| Copy token & DB to 2nd node | Bind `sub`/`tid` to host fingerprint (TPM EK) – optional enterprise control | +| Counter DB rollback | Hash‑chain + monotonic clock – optional enterprise control | +| Flooding single node | Redis‑backed cluster rate‑limit (30 hits / 60 s) + edge Nginx (20 r/s) | +| Key compromise | Rotate RS256 key‑pair, ship new pubkey, re‑sign tokens | + +--- + +## 8  Anonymous (33 runs) mode + +Offline PoCs without registration still work: + +```bash +docker compose exec stella-ops stella-jwt reload # reloads, discovers no token +``` + +…but **production deployments *must* register** to unlock real‑world quotas and +receive security advisories via e‑mail. + +--- + +*Last updated: 2025‑08‑02* \ No newline at end of file diff --git a/global.json b/global.json new file mode 100644 index 00000000..56e246dd --- /dev/null +++ b/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "version": "10.0.100-preview.7.25380.108", + "rollForward": "latestMinor" + } +} diff --git a/scripts/render_docs.py b/scripts/render_docs.py new file mode 100644 index 00000000..efefbb03 --- /dev/null +++ b/scripts/render_docs.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Render Markdown documentation under docs/ into a static HTML bundle. + +The script converts every Markdown file into a standalone HTML document, +mirroring the original folder structure under the output directory. A +`manifest.json` file is also produced to list the generated documents and +surface basic metadata (title, source path, output path). + +Usage: + python scripts/render_docs.py --source docs --output build/docs-site + +Dependencies: + pip install markdown pygments +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import shutil +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable, List + +import markdown + +# Enable fenced code blocks, tables, and definition lists. These cover the +# Markdown constructs heavily used across the documentation set. +MD_EXTENSIONS = [ + "fenced_code", + "codehilite", + "tables", + "toc", + "def_list", + "admonition", +] + +HTML_TEMPLATE = """ + + + + + {title} + + + +
+{body} +
+
+

Generated on {generated_at} UTC · Source: {source}

+
+ + +""" + + +@dataclass +class DocEntry: + source: Path + output: Path + title: str + + def to_manifest(self) -> dict[str, str]: + return { + "source": self.source.as_posix(), + "output": self.output.as_posix(), + "title": self.title, + } + + +def discover_markdown_files(source_root: Path) -> Iterable[Path]: + for path in source_root.rglob("*.md"): + if path.is_file(): + yield path + + +def read_title(markdown_text: str, fallback: str) -> str: + for raw_line in markdown_text.splitlines(): + line = raw_line.strip() + if line.startswith("#"): + return line.lstrip("#").strip() or fallback + return fallback + + +def convert_markdown(path: Path, source_root: Path, output_root: Path) -> DocEntry: + relative = path.relative_to(source_root) + output_path = output_root / relative.with_suffix(".html") + output_path.parent.mkdir(parents=True, exist_ok=True) + + text = path.read_text(encoding="utf-8") + html_body = markdown.markdown(text, extensions=MD_EXTENSIONS) + + title = read_title(text, fallback=relative.stem.replace("_", " ")) + generated_at = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + output_path.write_text( + HTML_TEMPLATE.format( + title=title, + body=html_body, + generated_at=generated_at, + source=relative.as_posix(), + ), + encoding="utf-8", + ) + + return DocEntry(source=relative, output=output_path.relative_to(output_root), title=title) + + +def copy_static_assets(source_root: Path, output_root: Path) -> None: + for path in source_root.rglob("*"): + if path.is_dir() or path.suffix.lower() == ".md": + # Skip Markdown (already rendered separately). + continue + relative = path.relative_to(source_root) + destination = output_root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(path.read_bytes()) + logging.info("Copied asset %s", relative) + + +def write_manifest(entries: Iterable[DocEntry], output_root: Path) -> None: + manifest_path = output_root / "manifest.json" + manifest = [entry.to_manifest() for entry in entries] + manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + logging.info("Wrote manifest with %d entries", len(manifest)) + + +def write_index(entries: List[DocEntry], output_root: Path) -> None: + index_path = output_root / "index.html" + generated_at = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + items = "\n".join( + f"
  • {entry.title}" f" · {entry.source.as_posix()}
    • " + for entry in sorted(entries, key=lambda e: e.title.lower()) + ) + + html = f""" + + + + + Stella Ops Documentation Index + + + +

    Stella Ops Documentation

    +

    Generated on {generated_at} UTC

    +
      +{items} +
    + + +""" + index_path.write_text(html, encoding="utf-8") + logging.info("Wrote HTML index with %d entries", len(entries)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Render documentation bundle") + parser.add_argument("--source", default="docs", type=Path, help="Directory containing Markdown sources") + parser.add_argument("--output", default=Path("build/docs-site"), type=Path, help="Directory for rendered output") + parser.add_argument("--clean", action="store_true", help="Remove the output directory before rendering") + return parser.parse_args() + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + args = parse_args() + + source_root: Path = args.source.resolve() + output_root: Path = args.output.resolve() + + if not source_root.exists(): + logging.error("Source directory %s does not exist", source_root) + return os.EX_NOINPUT + + if args.clean and output_root.exists(): + logging.info("Cleaning existing output directory %s", output_root) + shutil.rmtree(output_root) + + output_root.mkdir(parents=True, exist_ok=True) + + entries: List[DocEntry] = [] + for md_file in discover_markdown_files(source_root): + entry = convert_markdown(md_file, source_root, output_root) + entries.append(entry) + logging.info("Rendered %s -> %s", entry.source, entry.output) + + write_manifest(entries, output_root) + write_index(entries, output_root) + copy_static_assets(source_root, output_root) + + logging.info("Documentation bundle available at %s", output_root) + return os.EX_OK + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/Directory.Build.props b/src/Directory.Build.props new file mode 100644 index 00000000..220dbd97 --- /dev/null +++ b/src/Directory.Build.props @@ -0,0 +1,29 @@ + + + $(SolutionDir)PluginBinaries + true + true + + + + + false + runtime + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets new file mode 100644 index 00000000..9e722da9 --- /dev/null +++ b/src/Directory.Build.targets @@ -0,0 +1,17 @@ + + + + $(FeedserPluginOutputRoot)\$(MSBuildProjectName) + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Jobs.cs b/src/Jobs.cs new file mode 100644 index 00000000..93353722 --- /dev/null +++ b/src/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +internal static class OracleJobKinds +{ + public const string Fetch = "source:vndr-oracle:fetch"; + public const string Parse = "source:vndr-oracle:parse"; + public const string Map = "source:vndr-oracle:map"; +} + +internal sealed class OracleFetchJob : IJob +{ + private readonly OracleConnector _connector; + + public OracleFetchJob(OracleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class OracleParseJob : IJob +{ + private readonly OracleConnector _connector; + + public OracleParseJob(OracleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class OracleMapJob : IJob +{ + private readonly OracleConnector _connector; + + public OracleMapJob(OracleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/OracleConnector.cs b/src/OracleConnector.cs new file mode 100644 index 00000000..8cdf6f89 --- /dev/null +++ b/src/OracleConnector.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Feedser.Source.Vndr.Oracle.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public sealed class OracleConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly OracleOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public OracleConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IPsirtFlagStore psirtFlagStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VndrOracleConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + var now = _timeProvider.GetUtcNow(); + + foreach (var uri in _options.AdvisoryUris) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var advisoryId = DeriveAdvisoryId(uri); + var title = advisoryId.Replace('-', ' '); + var published = now; + + var metadata = OracleDocumentMetadata.CreateMetadata(advisoryId, title, published); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest(OracleOptions.HttpClientName, SourceName, uri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle fetch failed for {Uri}", uri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastProcessed(now); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Oracle document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + OracleDto dto; + try + { + var metadata = OracleDocumentMetadata.FromDocument(document); + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = System.Text.Encoding.UTF8.GetString(content); + dto = OracleParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = BsonDocument.Parse(json); + var validatedAt = _timeProvider.GetUtcNow(); + + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "oracle.advisory.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "oracle.advisory.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + OracleDto? dto; + try + { + var json = dtoRecord.Payload.ToJson(); + dto = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle DTO deserialization failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("Oracle DTO payload deserialized as null for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var mappedAt = _timeProvider.GetUtcNow(); + var (advisory, flag) = OracleMapper.Map(dto, SourceName, mappedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return OracleCursor.FromBson(record?.Cursor); + } + + private async Task UpdateCursorAsync(OracleCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private static string DeriveAdvisoryId(Uri uri) + { + var segments = uri.Segments; + if (segments.Length == 0) + { + return uri.AbsoluteUri; + } + + var slug = segments[^1].Trim('/'); + if (string.IsNullOrWhiteSpace(slug)) + { + return uri.AbsoluteUri; + } + + return slug.Replace('.', '-'); + } +} diff --git a/src/OracleConnectorPlugin.cs b/src/OracleConnectorPlugin.cs new file mode 100644 index 00000000..d22c9c27 --- /dev/null +++ b/src/OracleConnectorPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public sealed class VndrOracleConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-oracle"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/OracleDependencyInjectionRoutine.cs b/src/OracleDependencyInjectionRoutine.cs new file mode 100644 index 00000000..63a748dd --- /dev/null +++ b/src/OracleDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public sealed class OracleDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:oracle"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddOracleConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, OracleJobKinds.Fetch, typeof(OracleFetchJob)); + EnsureJob(options, OracleJobKinds.Parse, typeof(OracleParseJob)); + EnsureJob(options, OracleJobKinds.Map, typeof(OracleMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.DependencyInjection/IDependencyInjectionRoutine.cs b/src/StellaOps.DependencyInjection/IDependencyInjectionRoutine.cs new file mode 100644 index 00000000..07e44186 --- /dev/null +++ b/src/StellaOps.DependencyInjection/IDependencyInjectionRoutine.cs @@ -0,0 +1,11 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace StellaOps.DependencyInjection; + +public interface IDependencyInjectionRoutine +{ + IServiceCollection Register( + IServiceCollection services, + IConfiguration configuration); +} \ No newline at end of file diff --git a/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj b/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj new file mode 100644 index 00000000..1d418d89 --- /dev/null +++ b/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj @@ -0,0 +1,14 @@ + + + + net10.0 + enable + enable + + + + + + + + \ No newline at end of file diff --git a/src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs b/src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs new file mode 100644 index 00000000..1d1616f8 --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs @@ -0,0 +1,483 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class JobCoordinatorTests +{ + [Fact] + public async Task TriggerAsync_RunCompletesSuccessfully() + { + var services = new ServiceCollection(); + services.AddTransient(); + services.AddLogging(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new InMemoryLeaseStore(); + var jobOptions = new JobSchedulerOptions + { + DefaultLeaseDuration = TimeSpan.FromSeconds(5), + DefaultTimeout = TimeSpan.FromSeconds(10), + }; + + var definition = new JobDefinition( + Kind: "test:run", + JobType: typeof(SuccessfulJob), + Timeout: TimeSpan.FromSeconds(5), + LeaseDuration: TimeSpan.FromSeconds(2), + CronExpression: null, + Enabled: true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, new Dictionary { ["foo"] = "bar" }, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + var completed = await jobStore.Completion.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(JobRunStatus.Succeeded, completed.Status); + await leaseStore.WaitForReleaseAsync(TimeSpan.FromSeconds(1)); + Assert.True(leaseStore.ReleaseCount > 0); + Assert.Equal("bar", completed.Parameters["foo"]); + } + + [Fact] + public async Task TriggerAsync_MarksRunFailed_WhenLeaseReleaseFails() + { + var services = new ServiceCollection(); + services.AddTransient(); + services.AddLogging(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new FailingLeaseStore + { + ThrowOnRelease = true, + }; + + var jobOptions = new JobSchedulerOptions + { + DefaultLeaseDuration = TimeSpan.FromSeconds(5), + DefaultTimeout = TimeSpan.FromSeconds(10), + }; + + var definition = new JobDefinition( + Kind: "test:run", + JobType: typeof(SuccessfulJob), + Timeout: TimeSpan.FromSeconds(5), + LeaseDuration: TimeSpan.FromSeconds(2), + CronExpression: null, + Enabled: true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, null, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + var completed = await jobStore.Completion.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(JobRunStatus.Failed, completed.Status); + Assert.NotNull(completed.Error); + Assert.Contains("Failed to release lease", completed.Error!, StringComparison.OrdinalIgnoreCase); + Assert.True(leaseStore.ReleaseAttempts > 0); + } + + [Fact] + public async Task TriggerAsync_MarksRunFailed_WhenLeaseHeartbeatFails() + { + var services = new ServiceCollection(); + services.AddTransient(); + services.AddLogging(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new FailingLeaseStore + { + ThrowOnHeartbeat = true, + }; + + var jobOptions = new JobSchedulerOptions + { + DefaultLeaseDuration = TimeSpan.FromSeconds(2), + DefaultTimeout = TimeSpan.FromSeconds(10), + }; + + var definition = new JobDefinition( + Kind: "test:heartbeat", + JobType: typeof(SlowJob), + Timeout: TimeSpan.FromSeconds(5), + LeaseDuration: TimeSpan.FromSeconds(2), + CronExpression: null, + Enabled: true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, null, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + var completed = await jobStore.Completion.Task.WaitAsync(TimeSpan.FromSeconds(6)); + Assert.Equal(JobRunStatus.Failed, completed.Status); + Assert.NotNull(completed.Error); + Assert.Contains("Failed to heartbeat lease", completed.Error!, StringComparison.OrdinalIgnoreCase); + Assert.True(leaseStore.HeartbeatCount > 0); + } + + [Fact] + public async Task TriggerAsync_ReturnsAlreadyRunning_WhenLeaseUnavailable() + { + var services = new ServiceCollection(); + services.AddTransient(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new InMemoryLeaseStore + { + NextLease = null, + }; + var jobOptions = new JobSchedulerOptions(); + var definition = new JobDefinition( + "test:run", + typeof(SuccessfulJob), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(2), + null, + true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, null, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.AlreadyRunning, result.Outcome); + Assert.False(jobStore.CreatedRuns.Any()); + } + + [Fact] + public async Task TriggerAsync_ReturnsInvalidParameters_ForUnsupportedPayload() + { + var services = new ServiceCollection(); + services.AddTransient(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new InMemoryLeaseStore(); + var jobOptions = new JobSchedulerOptions(); + var definition = new JobDefinition( + "test:run", + typeof(SuccessfulJob), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(2), + null, + true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var parameters = new Dictionary + { + ["bad"] = new object(), + }; + + var result = await coordinator.TriggerAsync(definition.Kind, parameters, "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.InvalidParameters, result.Outcome); + Assert.Contains("unsupported", result.ErrorMessage, StringComparison.OrdinalIgnoreCase); + Assert.False(jobStore.CreatedRuns.Any()); + } + + [Fact] + public async Task TriggerAsync_CancelsJobOnTimeout() + { + var services = new ServiceCollection(); + services.AddTransient(); + using var provider = services.BuildServiceProvider(); + + var jobStore = new InMemoryJobStore(); + var leaseStore = new InMemoryLeaseStore(); + var jobOptions = new JobSchedulerOptions + { + DefaultLeaseDuration = TimeSpan.FromSeconds(5), + DefaultTimeout = TimeSpan.FromMilliseconds(100), + }; + + var definition = new JobDefinition( + Kind: "test:timeout", + JobType: typeof(TimeoutJob), + Timeout: TimeSpan.FromMilliseconds(100), + LeaseDuration: TimeSpan.FromSeconds(2), + CronExpression: null, + Enabled: true); + jobOptions.Definitions.Add(definition.Kind, definition); + + using var diagnostics = new JobDiagnostics(); + var coordinator = new JobCoordinator( + Options.Create(jobOptions), + jobStore, + leaseStore, + provider.GetRequiredService(), + NullLogger.Instance, + NullLoggerFactory.Instance, + new TestTimeProvider(), + diagnostics); + + var result = await coordinator.TriggerAsync(definition.Kind, null, "unit-test", CancellationToken.None); + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + + var completed = await jobStore.Completion.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(JobRunStatus.Cancelled, completed.Status); + await leaseStore.WaitForReleaseAsync(TimeSpan.FromSeconds(1)); + Assert.True(leaseStore.ReleaseCount > 0); + } + + private sealed class SuccessfulJob : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } + + private sealed class TimeoutJob : IJob + { + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + } + } + + private sealed class SlowJob : IJob + { + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); + } + } + + private sealed class InMemoryJobStore : IJobStore + { + private readonly Dictionary _runs = new(); + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public List CreatedRuns { get; } = new(); + + public Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken) + { + var run = new JobRunSnapshot( + Guid.NewGuid(), + request.Kind, + JobRunStatus.Pending, + request.CreatedAt, + null, + null, + request.Trigger, + request.ParametersHash, + null, + request.Timeout, + request.LeaseDuration, + request.Parameters); + _runs[run.RunId] = run; + CreatedRuns.Add(run); + return Task.FromResult(run); + } + + public Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) + { + if (_runs.TryGetValue(runId, out var run)) + { + var updated = run with { Status = JobRunStatus.Running, StartedAt = startedAt }; + _runs[runId] = updated; + return Task.FromResult(updated); + } + + return Task.FromResult(null); + } + + public Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) + { + if (_runs.TryGetValue(runId, out var run)) + { + var updated = run with { Status = completion.Status, CompletedAt = completion.CompletedAt, Error = completion.Error }; + _runs[runId] = updated; + Completion.TrySetResult(updated); + return Task.FromResult(updated); + } + + return Task.FromResult(null); + } + + public Task FindAsync(Guid runId, CancellationToken cancellationToken) + { + _runs.TryGetValue(runId, out var run); + return Task.FromResult(run); + } + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + { + var query = _runs.Values.AsEnumerable(); + if (!string.IsNullOrWhiteSpace(kind)) + { + query = query.Where(r => r.Kind == kind); + } + + return Task.FromResult>(query.OrderByDescending(r => r.CreatedAt).Take(limit).ToArray()); + } + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + { + return Task.FromResult>(_runs.Values.Where(r => r.Status is JobRunStatus.Pending or JobRunStatus.Running).ToArray()); + } + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + { + var run = _runs.Values + .Where(r => r.Kind == kind) + .OrderByDescending(r => r.CreatedAt) + .FirstOrDefault(); + return Task.FromResult(run); + } + + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + { + var results = new Dictionary(StringComparer.Ordinal); + foreach (var kind in kinds.Distinct(StringComparer.Ordinal)) + { + var run = _runs.Values + .Where(r => r.Kind == kind) + .OrderByDescending(r => r.CreatedAt) + .FirstOrDefault(); + if (run is not null) + { + results[kind] = run; + } + } + + return Task.FromResult>(results); + } + } + + private sealed class InMemoryLeaseStore : ILeaseStore + { + public JobLease? NextLease { get; set; } = new JobLease("job:test:run", "holder", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, TimeSpan.FromSeconds(2), DateTimeOffset.UtcNow.AddSeconds(2)); + public int HeartbeatCount { get; private set; } + public int ReleaseCount { get; private set; } + private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task TryAcquireAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + return Task.FromResult(NextLease); + } + + public Task HeartbeatAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + HeartbeatCount++; + NextLease = new JobLease(key, holder, now, now, leaseDuration, now.Add(leaseDuration)); + return Task.FromResult(NextLease); + } + + public Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken) + { + ReleaseCount++; + _released.TrySetResult(true); + return Task.FromResult(true); + } + + public Task WaitForReleaseAsync(TimeSpan timeout) + => _released.Task.WaitAsync(timeout); + } + + private sealed class FailingLeaseStore : ILeaseStore + { + public bool ThrowOnHeartbeat { get; set; } + public bool ThrowOnRelease { get; set; } + + public int HeartbeatCount { get; private set; } + public int ReleaseAttempts { get; private set; } + + public Task TryAcquireAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + var lease = new JobLease(key, holder, now, now, leaseDuration, now.Add(leaseDuration)); + return Task.FromResult(lease); + } + + public Task HeartbeatAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + HeartbeatCount++; + if (ThrowOnHeartbeat) + { + throw new InvalidOperationException("Lease heartbeat failed"); + } + + var lease = new JobLease(key, holder, now, now, leaseDuration, now.Add(leaseDuration)); + return Task.FromResult(lease); + } + + public Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken) + { + ReleaseAttempts++; + if (ThrowOnRelease) + { + throw new InvalidOperationException("Failed to release lease"); + } + + return Task.FromResult(true); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now = DateTimeOffset.Parse("2024-01-01T00:00:00Z"); + + public override DateTimeOffset GetUtcNow() => _now = _now.AddMilliseconds(100); + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs b/src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs new file mode 100644 index 00000000..0dc7fdae --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Plugin.Hosting; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class JobPluginRegistrationExtensionsTests +{ + [Fact] + public void RegisterJobPluginRoutines_LoadsPluginsAndRegistersDefinitions() + { + var services = new ServiceCollection(); + services.AddJobScheduler(); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["plugin:test:timeoutSeconds"] = "45", + }) + .Build(); + + var assemblyPath = typeof(JobPluginRegistrationExtensionsTests).Assembly.Location; + var pluginDirectory = Path.GetDirectoryName(assemblyPath)!; + var pluginFile = Path.GetFileName(assemblyPath); + + var options = new PluginHostOptions + { + BaseDirectory = pluginDirectory, + PluginsDirectory = pluginDirectory, + EnsureDirectoryExists = false, + RecursiveSearch = false, + }; + options.SearchPatterns.Add(pluginFile); + + services.RegisterJobPluginRoutines(configuration, options); + + Assert.Contains( + services, + descriptor => descriptor.ServiceType == typeof(PluginHostResult)); + + Assert.Contains( + services, + descriptor => descriptor.ServiceType.FullName == typeof(PluginRoutineExecuted).FullName); + + using var provider = services.BuildServiceProvider(); + var schedulerOptions = provider.GetRequiredService>().Value; + + Assert.True(schedulerOptions.Definitions.TryGetValue(PluginJob.JobKind, out var definition)); + Assert.NotNull(definition); + Assert.Equal(PluginJob.JobKind, definition.Kind); + Assert.Equal("StellaOps.Feedser.Core.Tests.PluginJob", definition.JobType.FullName); + Assert.Equal(TimeSpan.FromSeconds(45), definition.Timeout); + Assert.Equal(TimeSpan.FromSeconds(5), definition.LeaseDuration); + Assert.Equal("*/10 * * * *", definition.CronExpression); + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs b/src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs new file mode 100644 index 00000000..5e6f6385 --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs @@ -0,0 +1,70 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class JobSchedulerBuilderTests +{ + [Fact] + public void AddJob_RegistersDefinitionWithExplicitMetadata() + { + var services = new ServiceCollection(); + var builder = services.AddJobScheduler(); + + builder.AddJob( + kind: "jobs:test", + cronExpression: "*/5 * * * *", + timeout: TimeSpan.FromMinutes(42), + leaseDuration: TimeSpan.FromMinutes(7), + enabled: false); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.True(options.Definitions.TryGetValue("jobs:test", out var definition)); + Assert.NotNull(definition); + Assert.Equal(typeof(TestJob), definition.JobType); + Assert.Equal(TimeSpan.FromMinutes(42), definition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(7), definition.LeaseDuration); + Assert.Equal("*/5 * * * *", definition.CronExpression); + Assert.False(definition.Enabled); + } + + [Fact] + public void AddJob_UsesDefaults_WhenOptionalMetadataExcluded() + { + var services = new ServiceCollection(); + var builder = services.AddJobScheduler(options => + { + options.DefaultTimeout = TimeSpan.FromSeconds(123); + options.DefaultLeaseDuration = TimeSpan.FromSeconds(45); + }); + + builder.AddJob(kind: "jobs:defaults"); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.True(options.Definitions.TryGetValue("jobs:defaults", out var definition)); + Assert.NotNull(definition); + Assert.Equal(typeof(DefaultedJob), definition.JobType); + Assert.Equal(TimeSpan.FromSeconds(123), definition.Timeout); + Assert.Equal(TimeSpan.FromSeconds(45), definition.LeaseDuration); + Assert.Null(definition.CronExpression); + Assert.True(definition.Enabled); + } + + private sealed class TestJob : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => Task.CompletedTask; + } + + private sealed class DefaultedJob : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => Task.CompletedTask; + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs b/src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs new file mode 100644 index 00000000..80744720 --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs @@ -0,0 +1,42 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Core.Tests; + +public sealed class TestPluginRoutine : IDependencyInjectionRoutine +{ + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + var builder = new JobSchedulerBuilder(services); + var timeoutSeconds = configuration.GetValue("plugin:test:timeoutSeconds") ?? 30; + + builder.AddJob( + PluginJob.JobKind, + cronExpression: "*/10 * * * *", + timeout: TimeSpan.FromSeconds(timeoutSeconds), + leaseDuration: TimeSpan.FromSeconds(5)); + + services.AddSingleton(); + return services; + } +} + +public sealed class PluginRoutineExecuted +{ +} + +public sealed class PluginJob : IJob +{ + public const string JobKind = "plugin:test"; + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj b/src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj new file mode 100644 index 00000000..a45857e5 --- /dev/null +++ b/src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + + + + + diff --git a/src/StellaOps.Feedser.Core/AGENTS.md b/src/StellaOps.Feedser.Core/AGENTS.md new file mode 100644 index 00000000..d33278ec --- /dev/null +++ b/src/StellaOps.Feedser.Core/AGENTS.md @@ -0,0 +1,32 @@ +# AGENTS +## Role +Job orchestration and lifecycle. Registers job definitions, schedules execution, triggers runs, reports status for connectors and exporters. +## Scope +- Contracts: IJob (execute with CancellationToken), JobRunStatus, JobTriggerOutcome/Result. +- Registration: JobSchedulerBuilder.AddJob(kind, cronExpression?, timeout?, leaseDuration?); options recorded in JobSchedulerOptions. +- Plugin host integration discovers IJob providers via registered IDependencyInjectionRoutine implementations. +- Coordination: start/stop, single-flight via storage locks/leases, run bookkeeping (status, timings, errors). +- Triggering: manual/cron/API; parameterized runs; idempotent rejection if already running. +- Surfacing: enumerate definitions, last run, recent runs, active runs to WebService endpoints. +## Participants +- WebService exposes REST endpoints for definitions, runs, active, and trigger. +- Storage.Mongo persists job definitions metadata, run documents, and leases (locks collection). +- Source connectors and Exporters implement IJob and are registered into the scheduler via DI and Plugin routines. +- Models/Merge/Export are invoked indirectly through jobs. +- Plugin host runtime loads dependency injection routines that register job definitions. +## Interfaces & contracts +- Kind naming: family:source:verb (e.g., nvd:fetch, redhat:map, export:trivy-db). +- Timeout and lease duration enforce cancellation and duplicate-prevention. +- TimeProvider used for deterministic timing in tests. +## In/Out of scope +In: job lifecycle, registration, trigger semantics, run metadata. +Out: business logic of connectors/exporters, HTTP handlers (owned by WebService). +## Observability & security expectations +- Metrics: job.run.started/succeeded/failed, job.durationMs, job.concurrent.rejected, job.alreadyRunning. +- Logs: kind, trigger, params hash, lease holder, outcome; redact params containing secrets. +- Honor CancellationToken early and often. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Core.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Core/Jobs/IJob.cs b/src/StellaOps.Feedser.Core/Jobs/IJob.cs new file mode 100644 index 00000000..3c7290c2 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/IJob.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public interface IJob +{ + Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs b/src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs new file mode 100644 index 00000000..bce0cb29 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs @@ -0,0 +1,18 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public interface IJobCoordinator +{ + Task TriggerAsync(string kind, IReadOnlyDictionary? parameters, string trigger, CancellationToken cancellationToken); + + Task> GetDefinitionsAsync(CancellationToken cancellationToken); + + Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken); + + Task> GetActiveRunsAsync(CancellationToken cancellationToken); + + Task GetRunAsync(Guid runId, CancellationToken cancellationToken); + + Task GetLastRunAsync(string kind, CancellationToken cancellationToken); + + Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs b/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs new file mode 100644 index 00000000..a0eeb0ba --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs @@ -0,0 +1,20 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public interface IJobStore +{ + Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken); + + Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken); + + Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken); + + Task FindAsync(Guid runId, CancellationToken cancellationToken); + + Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken); + + Task> GetActiveRunsAsync(CancellationToken cancellationToken); + + Task GetLastRunAsync(string kind, CancellationToken cancellationToken); + + Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs b/src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs new file mode 100644 index 00000000..ead4b040 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public interface ILeaseStore +{ + Task TryAcquireAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken); + + Task HeartbeatAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken); + + Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs b/src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs new file mode 100644 index 00000000..7ad77184 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs @@ -0,0 +1,635 @@ +using System.Collections; +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobCoordinator : IJobCoordinator +{ + private readonly JobSchedulerOptions _options; + private readonly IJobStore _jobStore; + private readonly ILeaseStore _leaseStore; + private readonly IServiceScopeFactory _scopeFactory; + private readonly ILogger _logger; + private readonly ILoggerFactory _loggerFactory; + private readonly TimeProvider _timeProvider; + private readonly JobDiagnostics _diagnostics; + private readonly string _holderId; + + public JobCoordinator( + IOptions optionsAccessor, + IJobStore jobStore, + ILeaseStore leaseStore, + IServiceScopeFactory scopeFactory, + ILogger logger, + ILoggerFactory loggerFactory, + TimeProvider timeProvider, + JobDiagnostics diagnostics) + { + _options = (optionsAccessor ?? throw new ArgumentNullException(nameof(optionsAccessor))).Value; + _jobStore = jobStore ?? throw new ArgumentNullException(nameof(jobStore)); + _leaseStore = leaseStore ?? throw new ArgumentNullException(nameof(leaseStore)); + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _holderId = BuildHolderId(); + } + + public async Task TriggerAsync(string kind, IReadOnlyDictionary? parameters, string trigger, CancellationToken cancellationToken) + { + using var triggerActivity = _diagnostics.StartTriggerActivity(kind, trigger); + + if (!_options.Definitions.TryGetValue(kind, out var definition)) + { + var result = JobTriggerResult.NotFound($"Job kind '{kind}' is not registered."); + triggerActivity?.SetStatus(ActivityStatusCode.Error, result.ErrorMessage); + triggerActivity?.SetTag("job.trigger.outcome", result.Outcome.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "not_found"); + return result; + } + + triggerActivity?.SetTag("job.enabled", definition.Enabled); + triggerActivity?.SetTag("job.timeout_seconds", definition.Timeout.TotalSeconds); + triggerActivity?.SetTag("job.lease_seconds", definition.LeaseDuration.TotalSeconds); + + if (!definition.Enabled) + { + var result = JobTriggerResult.Disabled($"Job kind '{kind}' is disabled."); + triggerActivity?.SetStatus(ActivityStatusCode.Ok, "disabled"); + triggerActivity?.SetTag("job.trigger.outcome", result.Outcome.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "disabled"); + return result; + } + + parameters ??= new Dictionary(); + + var parameterSnapshot = parameters.Count == 0 + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(parameters, StringComparer.Ordinal); + + if (!TryNormalizeParameters(parameterSnapshot, out var normalizedParameters, out var parameterError)) + { + var message = string.IsNullOrWhiteSpace(parameterError) + ? "Job trigger parameters contain unsupported values." + : parameterError; + triggerActivity?.SetStatus(ActivityStatusCode.Error, message); + triggerActivity?.SetTag("job.trigger.outcome", JobTriggerOutcome.InvalidParameters.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "invalid_parameters"); + return JobTriggerResult.InvalidParameters(message); + } + + parameterSnapshot = normalizedParameters; + + string? parametersHash; + try + { + parametersHash = JobParametersHasher.Compute(parameterSnapshot); + } + catch (Exception ex) + { + var message = $"Job trigger parameters cannot be serialized: {ex.Message}"; + triggerActivity?.SetStatus(ActivityStatusCode.Error, message); + triggerActivity?.SetTag("job.trigger.outcome", JobTriggerOutcome.InvalidParameters.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "invalid_parameters"); + _logger.LogWarning(ex, "Failed to serialize parameters for job {Kind}", kind); + return JobTriggerResult.InvalidParameters(message); + } + + triggerActivity?.SetTag("job.parameters_count", parameterSnapshot.Count); + + var now = _timeProvider.GetUtcNow(); + var leaseDuration = definition.LeaseDuration <= TimeSpan.Zero ? _options.DefaultLeaseDuration : definition.LeaseDuration; + + JobLease? lease = null; + try + { + lease = await _leaseStore.TryAcquireAsync(definition.LeaseKey, _holderId, leaseDuration, now, cancellationToken).ConfigureAwait(false); + if (lease is null) + { + var result = JobTriggerResult.AlreadyRunning($"Job '{kind}' is already running."); + triggerActivity?.SetStatus(ActivityStatusCode.Ok, "already_running"); + triggerActivity?.SetTag("job.trigger.outcome", result.Outcome.ToString()); + _diagnostics.RecordTriggerRejected(kind, trigger, "already_running"); + return result; + } + + var createdAt = _timeProvider.GetUtcNow(); + var request = new JobRunCreateRequest( + definition.Kind, + trigger, + parameterSnapshot, + parametersHash, + definition.Timeout, + leaseDuration, + createdAt); + + triggerActivity?.SetTag("job.parameters_hash", request.ParametersHash); + + var run = await _jobStore.CreateAsync(request, cancellationToken).ConfigureAwait(false); + var startedAt = _timeProvider.GetUtcNow(); + var started = await _jobStore.TryStartAsync(run.RunId, startedAt, cancellationToken).ConfigureAwait(false) ?? run; + + triggerActivity?.SetTag("job.run_id", started.RunId); + triggerActivity?.SetTag("job.created_at", createdAt.UtcDateTime); + triggerActivity?.SetTag("job.started_at", started.StartedAt?.UtcDateTime ?? startedAt.UtcDateTime); + + var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + if (definition.Timeout > TimeSpan.Zero) + { + linkedTokenSource.CancelAfter(definition.Timeout); + } + + var capturedLease = lease ?? throw new InvalidOperationException("Lease acquisition returned null."); + try + { + _ = Task.Run(() => ExecuteJobAsync(definition, capturedLease, started, parameterSnapshot, trigger, linkedTokenSource), CancellationToken.None) + .ContinueWith(t => + { + if (t.Exception is not null) + { + _logger.LogError(t.Exception, "Unhandled job execution failure for {Kind}", definition.Kind); + } + }, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously); + lease = null; // released by background job execution + } + catch (Exception ex) + { + lease = capturedLease; // ensure outer finally releases if scheduling fails + triggerActivity?.SetStatus(ActivityStatusCode.Error, ex.Message); + triggerActivity?.SetTag("job.trigger.outcome", "exception"); + _diagnostics.RecordTriggerRejected(kind, trigger, "queue_failure"); + throw; + } + + var accepted = JobTriggerResult.Accepted(started); + _diagnostics.RecordTriggerAccepted(kind, trigger); + triggerActivity?.SetStatus(ActivityStatusCode.Ok); + triggerActivity?.SetTag("job.trigger.outcome", accepted.Outcome.ToString()); + return accepted; + } + catch (Exception ex) + { + triggerActivity?.SetStatus(ActivityStatusCode.Error, ex.Message); + triggerActivity?.SetTag("job.trigger.outcome", "exception"); + _diagnostics.RecordTriggerRejected(kind, trigger, "exception"); + throw; + } + finally + { + // Release handled by background execution path. If we failed before scheduling, release here. + if (lease is not null) + { + var releaseError = await TryReleaseLeaseAsync(lease, definition.Kind).ConfigureAwait(false); + if (releaseError is not null) + { + _logger.LogError(releaseError, "Failed to release lease {LeaseKey} for job {Kind}", lease.Key, definition.Kind); + } + } + } + } + + public Task> GetDefinitionsAsync(CancellationToken cancellationToken) + { + IReadOnlyList results = _options.Definitions.Values.OrderBy(x => x.Kind, StringComparer.Ordinal).ToArray(); + return Task.FromResult(results); + } + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + => _jobStore.GetRecentRunsAsync(kind, limit, cancellationToken); + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + => _jobStore.GetActiveRunsAsync(cancellationToken); + + public Task GetRunAsync(Guid runId, CancellationToken cancellationToken) + => _jobStore.FindAsync(runId, cancellationToken); + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + => _jobStore.GetLastRunAsync(kind, cancellationToken); + + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + => _jobStore.GetLastRunsAsync(kinds, cancellationToken); + + private static bool TryNormalizeParameters( + IReadOnlyDictionary source, + out Dictionary normalized, + out string? error) + { + if (source.Count == 0) + { + normalized = new Dictionary(StringComparer.Ordinal); + error = null; + return true; + } + + normalized = new Dictionary(source.Count, StringComparer.Ordinal); + foreach (var kvp in source) + { + if (string.IsNullOrWhiteSpace(kvp.Key)) + { + error = "Parameter keys must be non-empty strings."; + normalized = default!; + return false; + } + + try + { + normalized[kvp.Key] = NormalizeParameterValue(kvp.Value); + } + catch (Exception ex) + { + error = $"Parameter '{kvp.Key}' cannot be serialized: {ex.Message}"; + normalized = default!; + return false; + } + } + + error = null; + return true; + } + + private static object? NormalizeParameterValue(object? value) + { + if (value is null) + { + return null; + } + + switch (value) + { + case string or bool or double or decimal: + return value; + case byte or sbyte or short or ushort or int or long: + return Convert.ToInt64(value, CultureInfo.InvariantCulture); + case uint ui: + return Convert.ToInt64(ui); + case ulong ul when ul <= long.MaxValue: + return (long)ul; + case ulong ul: + return ul.ToString(CultureInfo.InvariantCulture); + case float f: + return (double)f; + case DateTime dt: + return dt.Kind == DateTimeKind.Utc ? dt : dt.ToUniversalTime(); + case DateTimeOffset dto: + return dto.ToUniversalTime(); + case TimeSpan ts: + return ts.ToString("c", CultureInfo.InvariantCulture); + case Guid guid: + return guid.ToString("D"); + case Enum enumValue: + return enumValue.ToString(); + case byte[] bytes: + return Convert.ToBase64String(bytes); + case JsonDocument document: + return NormalizeJsonElement(document.RootElement); + case JsonElement element: + return NormalizeJsonElement(element); + case IDictionary dictionary: + { + var nested = new SortedDictionary(StringComparer.Ordinal); + foreach (DictionaryEntry entry in dictionary) + { + if (entry.Key is not string key || string.IsNullOrWhiteSpace(key)) + { + throw new InvalidOperationException("Nested dictionary keys must be non-empty strings."); + } + + nested[key] = NormalizeParameterValue(entry.Value); + } + + return nested; + } + case IEnumerable enumerable when value is not string: + { + var list = new List(); + foreach (var item in enumerable) + { + list.Add(NormalizeParameterValue(item)); + } + + return list; + } + default: + throw new InvalidOperationException($"Unsupported parameter value of type '{value.GetType().FullName}'."); + } + } + + private static object? NormalizeJsonElement(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.Null => null, + JsonValueKind.String => element.GetString(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Number => element.TryGetInt64(out var l) + ? l + : element.TryGetDecimal(out var dec) + ? dec + : element.GetDouble(), + JsonValueKind.Object => NormalizeJsonObject(element), + JsonValueKind.Array => NormalizeJsonArray(element), + _ => throw new InvalidOperationException($"Unsupported JSON value '{element.ValueKind}'."), + }; + } + + private static SortedDictionary NormalizeJsonObject(JsonElement element) + { + var result = new SortedDictionary(StringComparer.Ordinal); + foreach (var property in element.EnumerateObject()) + { + result[property.Name] = NormalizeJsonElement(property.Value); + } + + return result; + } + + private static List NormalizeJsonArray(JsonElement element) + { + var items = new List(); + foreach (var item in element.EnumerateArray()) + { + items.Add(NormalizeJsonElement(item)); + } + + return items; + } + + private async Task CompleteRunAsync(Guid runId, JobRunStatus status, string? error, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + var completion = new JobRunCompletion(status, completedAt, error); + return await _jobStore.TryCompleteAsync(runId, completion, cancellationToken).ConfigureAwait(false); + } + + private TimeSpan? ResolveDuration(JobRunSnapshot original, JobRunSnapshot? completed) + { + if (completed?.Duration is { } duration) + { + return duration; + } + + var startedAt = completed?.StartedAt ?? original.StartedAt ?? original.CreatedAt; + var completedAt = completed?.CompletedAt ?? _timeProvider.GetUtcNow(); + var elapsed = completedAt - startedAt; + return elapsed >= TimeSpan.Zero ? elapsed : null; + } + + private static async Task ObserveLeaseTaskAsync(Task heartbeatTask) + { + try + { + await heartbeatTask.ConfigureAwait(false); + return null; + } + catch (OperationCanceledException) + { + return null; + } + catch (Exception ex) + { + return ex; + } + } + + private async Task TryReleaseLeaseAsync(JobLease lease, string kind) + { + try + { + await _leaseStore.ReleaseAsync(lease.Key, _holderId, CancellationToken.None).ConfigureAwait(false); + return null; + } + catch (Exception ex) + { + return new LeaseMaintenanceException($"Failed to release lease for job '{kind}'.", ex); + } + } + + private static Exception? CombineLeaseExceptions(Exception? first, Exception? second) + { + if (first is null) + { + return second; + } + + if (second is null) + { + return first; + } + + return new AggregateException(first, second); + } + + private async Task ExecuteJobAsync( + JobDefinition definition, + JobLease lease, + JobRunSnapshot run, + IReadOnlyDictionary parameters, + string trigger, + CancellationTokenSource linkedTokenSource) + { + using (linkedTokenSource) + { + var cancellationToken = linkedTokenSource.Token; + using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var heartbeatTask = MaintainLeaseAsync(definition, lease, heartbeatCts.Token); + + using var activity = _diagnostics.StartExecutionActivity(run.Kind, trigger, run.RunId); + activity?.SetTag("job.timeout_seconds", definition.Timeout.TotalSeconds); + activity?.SetTag("job.lease_seconds", definition.LeaseDuration.TotalSeconds); + activity?.SetTag("job.parameters_count", parameters.Count); + activity?.SetTag("job.created_at", run.CreatedAt.UtcDateTime); + activity?.SetTag("job.started_at", (run.StartedAt ?? run.CreatedAt).UtcDateTime); + activity?.SetTag("job.parameters_hash", run.ParametersHash); + + _diagnostics.RecordRunStarted(run.Kind); + + JobRunStatus finalStatus = JobRunStatus.Succeeded; + string? error = null; + Exception? executionException = null; + JobRunSnapshot? completedSnapshot = null; + Exception? leaseException = null; + + try + { + using var scope = _scopeFactory.CreateScope(); + var job = (IJob)scope.ServiceProvider.GetRequiredService(definition.JobType); + var jobLogger = _loggerFactory.CreateLogger(definition.JobType); + + var context = new JobExecutionContext( + run.RunId, + run.Kind, + trigger, + parameters, + scope.ServiceProvider, + _timeProvider, + jobLogger); + + await job.ExecuteAsync(context, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException oce) + { + finalStatus = JobRunStatus.Cancelled; + error = oce.Message; + executionException = oce; + } + catch (Exception ex) + { + finalStatus = JobRunStatus.Failed; + error = ex.ToString(); + executionException = ex; + } + finally + { + heartbeatCts.Cancel(); + + leaseException = await ObserveLeaseTaskAsync(heartbeatTask).ConfigureAwait(false); + + var releaseException = await TryReleaseLeaseAsync(lease, definition.Kind).ConfigureAwait(false); + leaseException = CombineLeaseExceptions(leaseException, releaseException); + + if (leaseException is not null) + { + var leaseMessage = $"Lease maintenance failed: {leaseException.GetType().Name}: {leaseException.Message}"; + if (finalStatus != JobRunStatus.Failed) + { + finalStatus = JobRunStatus.Failed; + error = leaseMessage; + executionException = leaseException; + } + else + { + error = string.IsNullOrWhiteSpace(error) + ? leaseMessage + : $"{error}{Environment.NewLine}{leaseMessage}"; + executionException = executionException is null + ? leaseException + : new AggregateException(executionException, leaseException); + } + } + } + + completedSnapshot = await CompleteRunAsync(run.RunId, finalStatus, error, CancellationToken.None).ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(error)) + { + activity?.SetTag("job.error", error); + } + + activity?.SetTag("job.status", finalStatus.ToString()); + + var completedDuration = ResolveDuration(run, completedSnapshot); + if (completedDuration.HasValue) + { + activity?.SetTag("job.duration_seconds", completedDuration.Value.TotalSeconds); + } + + switch (finalStatus) + { + case JobRunStatus.Succeeded: + activity?.SetStatus(ActivityStatusCode.Ok); + _logger.LogInformation("Job {Kind} run {RunId} succeeded", run.Kind, run.RunId); + break; + case JobRunStatus.Cancelled: + activity?.SetStatus(ActivityStatusCode.Ok, "cancelled"); + _logger.LogWarning(executionException, "Job {Kind} run {RunId} cancelled", run.Kind, run.RunId); + break; + case JobRunStatus.Failed: + activity?.SetStatus(ActivityStatusCode.Error, executionException?.Message ?? error); + _logger.LogError(executionException, "Job {Kind} run {RunId} failed", run.Kind, run.RunId); + break; + } + + _diagnostics.RecordRunCompleted(run.Kind, finalStatus, completedDuration, error); + } + } + + private async Task MaintainLeaseAsync(JobDefinition definition, JobLease lease, CancellationToken cancellationToken) + { + var leaseDuration = lease.LeaseDuration <= TimeSpan.Zero ? _options.DefaultLeaseDuration : lease.LeaseDuration; + var delay = TimeSpan.FromMilliseconds(Math.Max(1000, leaseDuration.TotalMilliseconds / 2)); + + while (!cancellationToken.IsCancellationRequested) + { + try + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + + var now = _timeProvider.GetUtcNow(); + try + { + await _leaseStore.HeartbeatAsync(definition.LeaseKey, _holderId, leaseDuration, now, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + throw new LeaseMaintenanceException($"Failed to heartbeat lease for job '{definition.Kind}'.", ex); + } + } + } + + private static string BuildHolderId() + { + var machine = Environment.MachineName; + var processId = Environment.ProcessId; + return $"{machine}:{processId}"; + } +} + +internal sealed class LeaseMaintenanceException : Exception +{ + public LeaseMaintenanceException(string message, Exception innerException) + : base(message, innerException) + { + } +} + +internal static class JobParametersHasher +{ + internal static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }; + + public static string? Compute(IReadOnlyDictionary parameters) + { + if (parameters is null || parameters.Count == 0) + { + return null; + } + + var canonicalJson = JsonSerializer.Serialize(Sort(parameters), SerializerOptions); + var bytes = Encoding.UTF8.GetBytes(canonicalJson); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static SortedDictionary Sort(IReadOnlyDictionary parameters) + { + var sorted = new SortedDictionary(StringComparer.Ordinal); + foreach (var kvp in parameters) + { + sorted[kvp.Key] = kvp.Value; + } + + return sorted; + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs b/src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs new file mode 100644 index 00000000..b822fced --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed record JobDefinition( + string Kind, + Type JobType, + TimeSpan Timeout, + TimeSpan LeaseDuration, + string? CronExpression, + bool Enabled) +{ + public string LeaseKey => $"job:{Kind}"; +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs b/src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs new file mode 100644 index 00000000..994ae6c9 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs @@ -0,0 +1,171 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobDiagnostics : IDisposable +{ + public const string ActivitySourceName = "StellaOps.Feedser.Jobs"; + public const string MeterName = "StellaOps.Feedser.Jobs"; + public const string TriggerActivityName = "feedser.job.trigger"; + public const string ExecuteActivityName = "feedser.job.execute"; + public const string SchedulerActivityName = "feedser.scheduler.evaluate"; + + private readonly Counter _triggersAccepted; + private readonly Counter _triggersRejected; + private readonly Counter _runsCompleted; + private readonly UpDownCounter _runsActive; + private readonly Histogram _runDurationSeconds; + private readonly Histogram _schedulerSkewMilliseconds; + + public JobDiagnostics() + { + ActivitySource = new ActivitySource(ActivitySourceName); + Meter = new Meter(MeterName); + + _triggersAccepted = Meter.CreateCounter( + name: "feedser.jobs.triggers.accepted", + unit: "count", + description: "Number of job trigger requests accepted for execution."); + + _triggersRejected = Meter.CreateCounter( + name: "feedser.jobs.triggers.rejected", + unit: "count", + description: "Number of job trigger requests rejected or ignored by the coordinator."); + + _runsCompleted = Meter.CreateCounter( + name: "feedser.jobs.runs.completed", + unit: "count", + description: "Number of job executions that have finished grouped by outcome."); + + _runsActive = Meter.CreateUpDownCounter( + name: "feedser.jobs.runs.active", + unit: "count", + description: "Current number of running job executions."); + + _runDurationSeconds = Meter.CreateHistogram( + name: "feedser.jobs.runs.duration", + unit: "s", + description: "Distribution of job execution durations in seconds."); + + _schedulerSkewMilliseconds = Meter.CreateHistogram( + name: "feedser.scheduler.skew", + unit: "ms", + description: "Difference between the intended and actual scheduler fire time in milliseconds."); + } + + public ActivitySource ActivitySource { get; } + + public Meter Meter { get; } + + public Activity? StartTriggerActivity(string kind, string trigger) + { + var activity = ActivitySource.StartActivity(TriggerActivityName, ActivityKind.Internal); + if (activity is not null) + { + activity.SetTag("job.kind", kind); + activity.SetTag("job.trigger", trigger); + } + + return activity; + } + + public Activity? StartSchedulerActivity(string kind, DateTimeOffset scheduledFor, DateTimeOffset invokedAt) + { + var activity = ActivitySource.StartActivity(SchedulerActivityName, ActivityKind.Internal); + if (activity is not null) + { + activity.SetTag("job.kind", kind); + activity.SetTag("job.scheduled_for", scheduledFor.UtcDateTime); + activity.SetTag("job.invoked_at", invokedAt.UtcDateTime); + activity.SetTag("job.scheduler_delay_ms", (invokedAt - scheduledFor).TotalMilliseconds); + } + + return activity; + } + + public Activity? StartExecutionActivity(string kind, string trigger, Guid runId) + { + var activity = ActivitySource.StartActivity(ExecuteActivityName, ActivityKind.Internal); + if (activity is not null) + { + activity.SetTag("job.kind", kind); + activity.SetTag("job.trigger", trigger); + activity.SetTag("job.run_id", runId); + } + + return activity; + } + + public void RecordTriggerAccepted(string kind, string trigger) + { + var tags = new TagList + { + { "job.kind", kind }, + { "job.trigger", trigger }, + }; + _triggersAccepted.Add(1, tags); + } + + public void RecordTriggerRejected(string kind, string trigger, string reason) + { + var tags = new TagList + { + { "job.kind", kind }, + { "job.trigger", trigger }, + { "job.reason", reason }, + }; + _triggersRejected.Add(1, tags); + } + + public void RecordRunStarted(string kind) + { + var tags = new TagList { { "job.kind", kind } }; + _runsActive.Add(1, tags); + } + + public void RecordRunCompleted(string kind, JobRunStatus status, TimeSpan? duration, string? error) + { + var outcome = status.ToString(); + + var completionTags = new TagList + { + { "job.kind", kind }, + { "job.status", outcome }, + }; + + if (!string.IsNullOrWhiteSpace(error)) + { + completionTags.Add("job.error", error); + } + + _runsCompleted.Add(1, completionTags); + + var activeTags = new TagList { { "job.kind", kind } }; + _runsActive.Add(-1, activeTags); + + if (duration.HasValue) + { + var seconds = Math.Max(duration.Value.TotalSeconds, 0d); + var durationTags = new TagList + { + { "job.kind", kind }, + { "job.status", outcome }, + }; + _runDurationSeconds.Record(seconds, durationTags); + } + } + + public void RecordSchedulerSkew(string kind, DateTimeOffset scheduledFor, DateTimeOffset invokedAt) + { + var skew = (invokedAt - scheduledFor).TotalMilliseconds; + var tags = new TagList { { "job.kind", kind } }; + _schedulerSkewMilliseconds.Record(skew, tags); + } + + public void Dispose() + { + ActivitySource.Dispose(); + Meter.Dispose(); + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs b/src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs new file mode 100644 index 00000000..300b2804 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobExecutionContext +{ + public JobExecutionContext( + Guid runId, + string kind, + string trigger, + IReadOnlyDictionary parameters, + IServiceProvider services, + TimeProvider timeProvider, + ILogger logger) + { + RunId = runId; + Kind = kind; + Trigger = trigger; + Parameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); + Services = services ?? throw new ArgumentNullException(nameof(services)); + TimeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Guid RunId { get; } + + public string Kind { get; } + + public string Trigger { get; } + + public IReadOnlyDictionary Parameters { get; } + + public IServiceProvider Services { get; } + + public TimeProvider TimeProvider { get; } + + public ILogger Logger { get; } + + public T GetRequiredService() where T : notnull + => Services.GetRequiredService(); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobLease.cs b/src/StellaOps.Feedser.Core/Jobs/JobLease.cs new file mode 100644 index 00000000..19b3991a --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobLease.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed record JobLease( + string Key, + string Holder, + DateTimeOffset AcquiredAt, + DateTimeOffset HeartbeatAt, + TimeSpan LeaseDuration, + DateTimeOffset TtlAt); diff --git a/src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs b/src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs new file mode 100644 index 00000000..47baee39 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.DependencyInjection; +using StellaOps.Plugin.Hosting; + +namespace StellaOps.Feedser.Core.Jobs; + +public static class JobPluginRegistrationExtensions +{ + public static IServiceCollection RegisterJobPluginRoutines( + this IServiceCollection services, + IConfiguration configuration, + PluginHostOptions options, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + ArgumentNullException.ThrowIfNull(options); + + var loadResult = PluginHost.LoadPlugins(options, logger); + + if (!services.Any(sd => sd.ServiceType == typeof(PluginHostResult))) + { + services.AddSingleton(loadResult); + } + + var currentServices = services; + var seenRoutineTypes = new HashSet(StringComparer.Ordinal); + + foreach (var plugin in loadResult.Plugins) + { + foreach (var routineType in GetRoutineTypes(plugin.Assembly)) + { + if (!typeof(IDependencyInjectionRoutine).IsAssignableFrom(routineType)) + { + continue; + } + + if (routineType.IsInterface || routineType.IsAbstract) + { + continue; + } + + var routineKey = routineType.FullName ?? routineType.Name; + if (!seenRoutineTypes.Add(routineKey)) + { + continue; + } + + IDependencyInjectionRoutine? routineInstance; + try + { + routineInstance = Activator.CreateInstance(routineType) as IDependencyInjectionRoutine; + } + catch (Exception ex) + { + logger?.LogWarning( + ex, + "Failed to create dependency injection routine {Routine} from plugin {Plugin}.", + routineType.FullName ?? routineType.Name, + plugin.Assembly.FullName ?? plugin.AssemblyPath); + continue; + } + + if (routineInstance is null) + { + continue; + } + + try + { + var updated = routineInstance.Register(currentServices, configuration); + if (updated is not null && !ReferenceEquals(updated, currentServices)) + { + currentServices = updated; + } + } + catch (Exception ex) + { + logger?.LogError( + ex, + "Dependency injection routine {Routine} from plugin {Plugin} threw during registration.", + routineType.FullName ?? routineType.Name, + plugin.Assembly.FullName ?? plugin.AssemblyPath); + } + } + } + + if (loadResult.MissingOrderedPlugins.Count > 0) + { + logger?.LogWarning( + "Missing ordered plugin(s): {Missing}", + string.Join(", ", loadResult.MissingOrderedPlugins)); + } + + return currentServices; + } + + private static IEnumerable GetRoutineTypes(Assembly assembly) + { + if (assembly is null) + { + yield break; + } + + Type[] types; + try + { + types = assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + types = ex.Types.Where(static t => t is not null)! + .Select(static t => t!) + .ToArray(); + } + + foreach (var type in types) + { + yield return type; + } + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs b/src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs new file mode 100644 index 00000000..965b6cd0 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed record JobRunCompletion( + JobRunStatus Status, + DateTimeOffset CompletedAt, + string? Error); diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs b/src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs new file mode 100644 index 00000000..c8993e8f --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed record JobRunCreateRequest( + string Kind, + string Trigger, + IReadOnlyDictionary Parameters, + string? ParametersHash, + TimeSpan? Timeout, + TimeSpan? LeaseDuration, + DateTimeOffset CreatedAt); diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs b/src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs new file mode 100644 index 00000000..d9672773 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs @@ -0,0 +1,21 @@ +namespace StellaOps.Feedser.Core.Jobs; + +/// +/// Immutable projection of a job run as stored in persistence. +/// +public sealed record JobRunSnapshot( + Guid RunId, + string Kind, + JobRunStatus Status, + DateTimeOffset CreatedAt, + DateTimeOffset? StartedAt, + DateTimeOffset? CompletedAt, + string Trigger, + string? ParametersHash, + string? Error, + TimeSpan? Timeout, + TimeSpan? LeaseDuration, + IReadOnlyDictionary Parameters) +{ + public TimeSpan? Duration => StartedAt is null || CompletedAt is null ? null : CompletedAt - StartedAt; +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs b/src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs new file mode 100644 index 00000000..7e3bcfe4 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public enum JobRunStatus +{ + Pending, + Running, + Succeeded, + Failed, + Cancelled, +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs new file mode 100644 index 00000000..9e396204 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs @@ -0,0 +1,47 @@ +using System; +using Microsoft.Extensions.DependencyInjection; + +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobSchedulerBuilder +{ + private readonly IServiceCollection _services; + + public JobSchedulerBuilder(IServiceCollection services) + { + _services = services ?? throw new ArgumentNullException(nameof(services)); + } + + public JobSchedulerBuilder AddJob( + string kind, + string? cronExpression = null, + TimeSpan? timeout = null, + TimeSpan? leaseDuration = null, + bool enabled = true) + where TJob : class, IJob + { + ArgumentException.ThrowIfNullOrEmpty(kind); + + _services.AddTransient(); + _services.Configure(options => + { + if (options.Definitions.ContainsKey(kind)) + { + throw new InvalidOperationException($"Job '{kind}' is already registered."); + } + + var resolvedTimeout = timeout ?? options.DefaultTimeout; + var resolvedLease = leaseDuration ?? options.DefaultLeaseDuration; + + options.Definitions.Add(kind, new JobDefinition( + kind, + typeof(TJob), + resolvedTimeout, + resolvedLease, + cronExpression, + enabled)); + }); + + return this; + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs new file mode 100644 index 00000000..6803b93d --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs @@ -0,0 +1,165 @@ +using Cronos; +using System.Diagnostics; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Feedser.Core.Jobs; + +/// +/// Background service that evaluates cron expressions for registered jobs and triggers them. +/// +public sealed class JobSchedulerHostedService : BackgroundService +{ + private readonly IJobCoordinator _coordinator; + private readonly JobSchedulerOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly JobDiagnostics _diagnostics; + private readonly Dictionary _cronExpressions = new(StringComparer.Ordinal); + private readonly Dictionary _nextOccurrences = new(StringComparer.Ordinal); + + public JobSchedulerHostedService( + IJobCoordinator coordinator, + IOptions optionsAccessor, + ILogger logger, + TimeProvider timeProvider, + JobDiagnostics diagnostics) + { + _coordinator = coordinator ?? throw new ArgumentNullException(nameof(coordinator)); + _options = (optionsAccessor ?? throw new ArgumentNullException(nameof(optionsAccessor))).Value; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + + foreach (var definition in _options.Definitions.Values) + { + if (string.IsNullOrWhiteSpace(definition.CronExpression)) + { + continue; + } + + try + { + var cron = CronExpression.Parse(definition.CronExpression!, CronFormat.Standard); + _cronExpressions[definition.Kind] = cron; + } + catch (CronFormatException ex) + { + _logger.LogError(ex, "Invalid cron expression '{Cron}' for job {Kind}", definition.CronExpression, definition.Kind); + } + } + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (_cronExpressions.Count == 0) + { + _logger.LogInformation("No cron-based jobs registered; scheduler idle."); + await Task.Delay(Timeout.Infinite, stoppingToken).ConfigureAwait(false); + return; + } + + while (!stoppingToken.IsCancellationRequested) + { + var now = _timeProvider.GetUtcNow(); + var nextWake = now.AddMinutes(5); // default sleep when nothing scheduled + + foreach (var (kind, cron) in _cronExpressions) + { + if (!_options.Definitions.TryGetValue(kind, out var definition) || !definition.Enabled) + { + continue; + } + + var next = GetNextOccurrence(kind, cron, now); + if (next <= now.AddMilliseconds(500)) + { + _ = TriggerJobAsync(kind, next, stoppingToken); + _nextOccurrences[kind] = GetNextOccurrence(kind, cron, now.AddSeconds(1)); + next = _nextOccurrences[kind]; + } + + if (next < nextWake) + { + nextWake = next; + } + } + + var delay = nextWake - now; + if (delay < TimeSpan.FromSeconds(1)) + { + delay = TimeSpan.FromSeconds(1); + } + + try + { + await Task.Delay(delay, stoppingToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + private DateTimeOffset GetNextOccurrence(string kind, CronExpression cron, DateTimeOffset reference) + { + if (_nextOccurrences.TryGetValue(kind, out var cached) && cached > reference) + { + return cached; + } + + var next = cron.GetNextOccurrence(reference.UtcDateTime, TimeZoneInfo.Utc); + if (next is null) + { + // No future occurrence; schedule far in future to avoid tight loop. + next = reference.UtcDateTime.AddYears(100); + } + + var nextUtc = DateTime.SpecifyKind(next.Value, DateTimeKind.Utc); + var offset = new DateTimeOffset(nextUtc); + _nextOccurrences[kind] = offset; + return offset; + } + + private async Task TriggerJobAsync(string kind, DateTimeOffset scheduledFor, CancellationToken stoppingToken) + { + var invokedAt = _timeProvider.GetUtcNow(); + _diagnostics.RecordSchedulerSkew(kind, scheduledFor, invokedAt); + + using var activity = _diagnostics.StartSchedulerActivity(kind, scheduledFor, invokedAt); + try + { + var result = await _coordinator.TriggerAsync(kind, parameters: null, trigger: "scheduler", stoppingToken).ConfigureAwait(false); + activity?.SetTag("job.trigger.outcome", result.Outcome.ToString()); + if (result.Run is not null) + { + activity?.SetTag("job.run_id", result.Run.RunId); + } + if (!string.IsNullOrWhiteSpace(result.ErrorMessage)) + { + activity?.SetTag("job.trigger.error", result.ErrorMessage); + } + + if (result.Outcome == JobTriggerOutcome.Accepted) + { + activity?.SetStatus(ActivityStatusCode.Ok); + } + else + { + activity?.SetStatus(ActivityStatusCode.Ok, result.Outcome.ToString()); + } + + if (result.Outcome != JobTriggerOutcome.Accepted) + { + _logger.LogDebug("Scheduler trigger for {Kind} resulted in {Outcome}", kind, result.Outcome); + } + } + catch (Exception ex) when (!stoppingToken.IsCancellationRequested) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + _logger.LogError(ex, "Cron trigger for job {Kind} failed", kind); + } + } +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs new file mode 100644 index 00000000..0ad51c08 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public sealed class JobSchedulerOptions +{ + public static JobSchedulerOptions Empty { get; } = new(); + + public IDictionary Definitions { get; } = new Dictionary(StringComparer.Ordinal); + + public TimeSpan DefaultTimeout { get; set; } = TimeSpan.FromMinutes(15); + + public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs b/src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs new file mode 100644 index 00000000..c8b33000 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs @@ -0,0 +1,40 @@ +namespace StellaOps.Feedser.Core.Jobs; + +public enum JobTriggerOutcome +{ + Accepted, + NotFound, + Disabled, + AlreadyRunning, + LeaseRejected, + InvalidParameters, + Failed, + Cancelled, +} + +public sealed record JobTriggerResult(JobTriggerOutcome Outcome, JobRunSnapshot? Run, string? ErrorMessage) +{ + public static JobTriggerResult Accepted(JobRunSnapshot run) + => new(JobTriggerOutcome.Accepted, run, null); + + public static JobTriggerResult NotFound(string message) + => new(JobTriggerOutcome.NotFound, null, message); + + public static JobTriggerResult Disabled(string message) + => new(JobTriggerOutcome.Disabled, null, message); + + public static JobTriggerResult AlreadyRunning(string message) + => new(JobTriggerOutcome.AlreadyRunning, null, message); + + public static JobTriggerResult LeaseRejected(string message) + => new(JobTriggerOutcome.LeaseRejected, null, message); + + public static JobTriggerResult InvalidParameters(string message) + => new(JobTriggerOutcome.InvalidParameters, null, message); + + public static JobTriggerResult Failed(JobRunSnapshot run, string error) + => new(JobTriggerOutcome.Failed, run, error); + + public static JobTriggerResult Cancelled(JobRunSnapshot run, string error) + => new(JobTriggerOutcome.Cancelled, run, error); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..a11822f2 --- /dev/null +++ b/src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; + +namespace StellaOps.Feedser.Core.Jobs; + +public static class JobServiceCollectionExtensions +{ + public static JobSchedulerBuilder AddJobScheduler(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + var optionsBuilder = services.AddOptions(); + if (configure is not null) + { + optionsBuilder.Configure(configure); + } + + services.AddSingleton(sp => sp.GetRequiredService>().Value); + services.AddSingleton(); + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(); + services.AddHostedService(); + + return new JobSchedulerBuilder(services); + } +} diff --git a/src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj b/src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj new file mode 100644 index 00000000..0655c8f3 --- /dev/null +++ b/src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Core/TASKS.md b/src/StellaOps.Feedser.Core/TASKS.md new file mode 100644 index 00000000..0712e0a7 --- /dev/null +++ b/src/StellaOps.Feedser.Core/TASKS.md @@ -0,0 +1,14 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|JobCoordinator implementation (create/get/mark status)|BE-Core|Storage.Mongo|DONE – `JobCoordinator` drives Mongo-backed runs.| +|Cron scheduling loop with TimeProvider|BE-Core|Core|DONE – `JobSchedulerHostedService` evaluates cron expressions.| +|Single-flight/lease semantics|BE-Core|Storage.Mongo|DONE – lease acquisition backed by `MongoLeaseStore`.| +|Trigger API contract (Result mapping)|BE-Core|WebService|DONE – `JobTriggerResult` outcomes map to HTTP statuses.| +|Run telemetry enrichment|BE-Core|Observability|DONE – `JobDiagnostics` ties activities & counters into coordinator/scheduler paths.| +|Deterministic params hashing|BE-Core|Core|DONE – `JobParametersHasher` creates SHA256 hash.| +|Golden tests for timeout/cancel|QA|Core|DONE – JobCoordinatorTests cover cancellation timeout path.| +|JobSchedulerBuilder options registry coverage|BE-Core|Core|DONE – added scheduler tests confirming cron/timeout/lease metadata persists via JobSchedulerOptions.| +|Plugin discovery + DI glue with PluginHost|BE-Core|Plugin libs|DONE – JobPluginRegistrationExtensions now loads PluginHost routines and wires connector/exporter registrations.| +|Harden lease release error handling in JobCoordinator|BE-Core|Storage.Mongo|DONE – lease release failures now logged, wrapped, and drive run failure status; fire-and-forget execution guarded. Verified with `dotnet test --no-build --filter JobCoordinator`.| +|Validate job trigger parameters for serialization|BE-Core|WebService|DONE – trigger parameters normalized/serialized with defensive checks returning InvalidParameters on failure. Full-suite `dotnet test --no-build` currently red from live connector fixture drift (Oracle/JVN/RedHat).| diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs new file mode 100644 index 00000000..ac2b39eb --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs @@ -0,0 +1,213 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class JsonExportSnapshotBuilderTests : IDisposable +{ + private readonly string _root; + + public JsonExportSnapshotBuilderTests() + { + _root = Directory.CreateTempSubdirectory("feedser-json-export-tests").FullName; + } + + [Fact] + public async Task WritesDeterministicTree() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture); + + var advisories = new[] + { + CreateAdvisory( + advisoryKey: "CVE-2024-9999", + aliases: new[] { "GHSA-zzzz-yyyy-xxxx", "CVE-2024-9999" }, + title: "Deterministic Snapshot", + severity: "critical"), + CreateAdvisory( + advisoryKey: "VENDOR-2024-42", + aliases: new[] { "ALIAS-1", "ALIAS-2" }, + title: "Vendor Advisory", + severity: "medium"), + }; + + var result = await builder.WriteAsync(advisories, exportedAt, cancellationToken: CancellationToken.None); + + Assert.Equal(advisories.Length, result.AdvisoryCount); + Assert.Equal(exportedAt, result.ExportedAt); + + var expectedFiles = result.FilePaths.OrderBy(x => x, StringComparer.Ordinal).ToArray(); + Assert.Contains("nvd/2024/CVE-2024-9999.json", expectedFiles); + Assert.Contains("misc/VENDOR-2024-42.json", expectedFiles); + + var cvePath = ResolvePath(result.ExportDirectory, "nvd/2024/CVE-2024-9999.json"); + Assert.True(File.Exists(cvePath)); + var actualJson = await File.ReadAllTextAsync(cvePath, CancellationToken.None); + Assert.Equal(SnapshotSerializer.ToSnapshot(advisories[0]), actualJson); + } + + [Fact] + public async Task ProducesIdenticalBytesAcrossRuns() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-05-01T00:00:00Z", CultureInfo.InvariantCulture); + var advisories = new[] + { + CreateAdvisory("CVE-2024-1000", new[] { "CVE-2024-1000", "GHSA-aaaa-bbbb-cccc" }, "Snapshot Stable", "high"), + }; + + var first = await builder.WriteAsync(advisories, exportedAt, exportName: "20240501T000000Z", CancellationToken.None); + var firstDigest = ComputeDigest(first); + + var second = await builder.WriteAsync(advisories, exportedAt, exportName: "20240501T000000Z", CancellationToken.None); + var secondDigest = ComputeDigest(second); + + Assert.Equal(Convert.ToHexString(firstDigest), Convert.ToHexString(secondDigest)); + } + + [Fact] + public async Task WriteAsync_NormalizesInputOrdering() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-06-01T00:00:00Z", CultureInfo.InvariantCulture); + + var advisoryA = CreateAdvisory("CVE-2024-1000", new[] { "CVE-2024-1000" }, "Alpha", "high"); + var advisoryB = CreateAdvisory("VENDOR-0001", new[] { "VENDOR-0001" }, "Vendor Advisory", "medium"); + + var result = await builder.WriteAsync(new[] { advisoryB, advisoryA }, exportedAt, cancellationToken: CancellationToken.None); + + var expectedOrder = result.FilePaths.OrderBy(path => path, StringComparer.Ordinal).ToArray(); + Assert.Equal(expectedOrder, result.FilePaths.ToArray()); + } + + [Fact] + public async Task WriteAsync_EnumeratesStreamOnlyOnce() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-08-01T00:00:00Z", CultureInfo.InvariantCulture); + + var advisories = new[] + { + CreateAdvisory("CVE-2024-2000", new[] { "CVE-2024-2000" }, "Streaming One", "medium"), + CreateAdvisory("CVE-2024-2001", new[] { "CVE-2024-2001" }, "Streaming Two", "low"), + }; + + var sequence = new SingleEnumerationAsyncSequence(advisories); + var result = await builder.WriteAsync(sequence, exportedAt, cancellationToken: CancellationToken.None); + + Assert.Equal(advisories.Length, result.AdvisoryCount); + } + + private static Advisory CreateAdvisory(string advisoryKey, string[] aliases, string title, string severity) + { + return new Advisory( + advisoryKey: advisoryKey, + title: title, + summary: null, + language: "EN", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: severity, + exploitKnown: false, + aliases: aliases, + references: new[] + { + new AdvisoryReference("https://example.com/advisory", "advisory", null, null, AdvisoryProvenance.Empty), + }, + affectedPackages: new[] + { + new AffectedPackage( + AffectedPackageTypes.SemVer, + "sample/package", + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: Array.Empty()), + }, + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("feedser", "normalized", "canonical", DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture)), + }); + } + + private static byte[] ComputeDigest(JsonExportResult result) + { + using var sha256 = SHA256.Create(); + foreach (var relative in result.FilePaths.OrderBy(x => x, StringComparer.Ordinal)) + { + var fullPath = ResolvePath(result.ExportDirectory, relative); + var bytes = File.ReadAllBytes(fullPath); + sha256.TransformBlock(bytes, 0, bytes.Length, null, 0); + } + + sha256.TransformFinalBlock(Array.Empty(), 0, 0); + return sha256.Hash ?? Array.Empty(); + } + + private static string ResolvePath(string root, string relative) + { + var segments = relative.Split('/', StringSplitOptions.RemoveEmptyEntries); + return Path.Combine(new[] { root }.Concat(segments).ToArray()); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + + private sealed class SingleEnumerationAsyncSequence : IAsyncEnumerable + { + private readonly IReadOnlyList _advisories; + private int _enumerated; + + public SingleEnumerationAsyncSequence(IReadOnlyList advisories) + { + _advisories = advisories ?? throw new ArgumentNullException(nameof(advisories)); + } + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + if (Interlocked.Exchange(ref _enumerated, 1) == 1) + { + throw new InvalidOperationException("Sequence was enumerated more than once."); + } + + return Enumerate(cancellationToken); + + async IAsyncEnumerator Enumerate([EnumeratorCancellation] CancellationToken ct) + { + foreach (var advisory in _advisories) + { + ct.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs new file mode 100644 index 00000000..64f0fbf7 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs @@ -0,0 +1,83 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class JsonExporterDependencyInjectionRoutineTests +{ + [Fact] + public void Register_AddsJobDefinitionAndServices() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(); + services.AddSingleton(); + services.AddOptions(); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary()) + .Build(); + + var routine = new JsonExporterDependencyInjectionRoutine(); + routine.Register(services, configuration); + + using var provider = services.BuildServiceProvider(); + var optionsAccessor = provider.GetRequiredService>(); + var options = optionsAccessor.Value; + + Assert.True(options.Definitions.TryGetValue(JsonExportJob.JobKind, out var definition)); + Assert.Equal(typeof(JsonExportJob), definition.JobType); + Assert.True(definition.Enabled); + + var exporter = provider.GetRequiredService(); + Assert.NotNull(exporter); + } + + private sealed class StubAdvisoryStore : IAdvisoryStore + { + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + => Task.FromResult>(Array.Empty()); + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + => Task.FromResult(null); + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + => Task.CompletedTask; + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) + { + return Enumerate(cancellationToken); + + static async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + await Task.Yield(); + yield break; + } + } + } + + private sealed class StubExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; + + public Task FindAsync(string id, CancellationToken cancellationToken) + => Task.FromResult(_record); + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs new file mode 100644 index 00000000..47795f7d --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs @@ -0,0 +1,182 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class JsonExporterParitySmokeTests : IDisposable +{ + private readonly string _root; + + public JsonExporterParitySmokeTests() + { + _root = Directory.CreateTempSubdirectory("feedser-json-parity-tests").FullName; + } + + [Fact] + public async Task ExportProducesVulnListCompatiblePaths() + { + var options = new JsonExportOptions { OutputRoot = _root }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportedAt = DateTimeOffset.Parse("2024-09-01T12:00:00Z", CultureInfo.InvariantCulture); + + var advisories = CreateSampleAdvisories(); + var result = await builder.WriteAsync(advisories, exportedAt, exportName: "parity-test", CancellationToken.None); + + var expected = new[] + { + "amazon/2/ALAS2-2024-1234.json", + "debian/DLA-2024-1234.json", + "ghsa/go/github.com%2Facme%2Fsample/GHSA-AAAA-BBBB-CCCC.json", + "nvd/2023/CVE-2023-27524.json", + "oracle/linux/ELSA-2024-12345.json", + "redhat/oval/RHSA-2024_0252.json", + "ubuntu/USN-6620-1.json", + "wolfi/WOLFI-2024-0001.json", + }; + + Assert.Equal(expected, result.FilePaths.ToArray()); + + foreach (var path in expected) + { + var fullPath = ResolvePath(result.ExportDirectory, path); + Assert.True(File.Exists(fullPath), $"Expected export file '{path}' to be present"); + } + } + + private static IReadOnlyList CreateSampleAdvisories() + { + var published = DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture); + var modified = DateTimeOffset.Parse("2024-02-01T00:00:00Z", CultureInfo.InvariantCulture); + + return new[] + { + CreateAdvisory( + "CVE-2023-27524", + "Apache Superset Improper Authentication", + new[] { "CVE-2023-27524" }, + null, + "nvd", + published, + modified), + CreateAdvisory( + "GHSA-aaaa-bbbb-cccc", + "Sample GHSA", + new[] { "CVE-2024-2000" }, + new[] + { + new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:go/github.com/acme/sample@1.0.0", + provenance: new[] { new AdvisoryProvenance("ghsa", "map", "", published) }) + }, + "ghsa", + published, + modified), + CreateAdvisory( + "USN-6620-1", + "Ubuntu Security Notice", + null, + null, + "ubuntu", + published, + modified), + CreateAdvisory( + "DLA-2024-1234", + "Debian LTS Advisory", + null, + null, + "debian", + published, + modified), + CreateAdvisory( + "RHSA-2024:0252", + "Red Hat Security Advisory", + null, + null, + "redhat", + published, + modified), + CreateAdvisory( + "ALAS2-2024-1234", + "Amazon Linux Advisory", + null, + null, + "amazon", + published, + modified), + CreateAdvisory( + "ELSA-2024-12345", + "Oracle Linux Advisory", + null, + null, + "oracle", + published, + modified), + CreateAdvisory( + "WOLFI-2024-0001", + "Wolfi Advisory", + null, + null, + "wolfi", + published, + modified), + }; + } + + private static Advisory CreateAdvisory( + string advisoryKey, + string title, + IEnumerable? aliases, + IEnumerable? packages, + string? provenanceSource, + DateTimeOffset? published, + DateTimeOffset? modified) + { + var provenance = provenanceSource is null + ? Array.Empty() + : new[] { new AdvisoryProvenance(provenanceSource, "normalize", "", modified ?? DateTimeOffset.UtcNow) }; + + return new Advisory( + advisoryKey, + title, + summary: null, + language: "en", + published, + modified, + severity: "medium", + exploitKnown: false, + aliases: aliases ?? Array.Empty(), + references: Array.Empty(), + affectedPackages: packages ?? Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: provenance); + } + + private static string ResolvePath(string root, string relative) + { + var segments = relative.Split('/', StringSplitOptions.RemoveEmptyEntries); + return Path.Combine(new[] { root }.Concat(segments).ToArray()); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs new file mode 100644 index 00000000..107714be --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs @@ -0,0 +1,265 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class JsonFeedExporterTests : IDisposable +{ + private readonly string _root; + + public JsonFeedExporterTests() + { + _root = Directory.CreateTempSubdirectory("feedser-json-exporter-tests").FullName; + } + + [Fact] + public async Task ExportAsync_SkipsWhenDigestUnchanged() + { + var advisory = new Advisory( + advisoryKey: "CVE-2024-1234", + title: "Test Advisory", + summary: null, + language: "en", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: "high", + exploitKnown: false, + aliases: new[] { "CVE-2024-1234" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var advisoryStore = new StubAdvisoryStore(advisory); + var options = Options.Create(new JsonExportOptions + { + OutputRoot = _root, + MaintainLatestSymlink = false, + }); + + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var exporter = new JsonFeedExporter( + advisoryStore, + options, + new VulnListJsonExportPathResolver(), + stateManager, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + var firstUpdated = record!.UpdatedAt; + Assert.Equal("20240715T120000Z", record.BaseExportId); + Assert.Equal(record.LastFullDigest, record.ExportCursor); + + var firstExportPath = Path.Combine(_root, "20240715T120000Z"); + Assert.True(Directory.Exists(firstExportPath)); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + await exporter.ExportAsync(provider, CancellationToken.None); + + record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal(firstUpdated, record!.UpdatedAt); + + var secondExportPath = Path.Combine(_root, "20240715T120500Z"); + Assert.False(Directory.Exists(secondExportPath)); + } + + [Fact] + public async Task ExportAsync_WritesManifestMetadata() + { + var exportedAt = DateTimeOffset.Parse("2024-08-10T00:00:00Z", CultureInfo.InvariantCulture); + var advisory = new Advisory( + advisoryKey: "CVE-2024-4321", + title: "Manifest Test", + summary: null, + language: "en", + published: DateTimeOffset.Parse("2024-07-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-07-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2024-4321" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var advisoryStore = new StubAdvisoryStore(advisory); + var optionsValue = new JsonExportOptions + { + OutputRoot = _root, + MaintainLatestSymlink = false, + }; + + var options = Options.Create(optionsValue); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(exportedAt); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var exporter = new JsonFeedExporter( + advisoryStore, + options, + new VulnListJsonExportPathResolver(), + stateManager, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture); + var exportDirectory = Path.Combine(_root, exportId); + var manifestPath = Path.Combine(exportDirectory, "manifest.json"); + + Assert.True(File.Exists(manifestPath)); + + using var document = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None)); + var root = document.RootElement; + + Assert.Equal(exportId, root.GetProperty("exportId").GetString()); + Assert.Equal(exportedAt.UtcDateTime, root.GetProperty("generatedAt").GetDateTime()); + Assert.Equal(1, root.GetProperty("advisoryCount").GetInt32()); + + var exportedFiles = Directory.EnumerateFiles(exportDirectory, "*.json", SearchOption.AllDirectories) + .Select(path => new + { + Absolute = path, + Relative = Path.GetRelativePath(exportDirectory, path).Replace("\\", "/", StringComparison.Ordinal), + }) + .Where(file => !string.Equals(file.Relative, "manifest.json", StringComparison.OrdinalIgnoreCase)) + .OrderBy(file => file.Relative, StringComparer.Ordinal) + .ToArray(); + + var filesElement = root.GetProperty("files") + .EnumerateArray() + .Select(element => new + { + Path = element.GetProperty("path").GetString(), + Bytes = element.GetProperty("bytes").GetInt64(), + Digest = element.GetProperty("digest").GetString(), + }) + .OrderBy(file => file.Path, StringComparer.Ordinal) + .ToArray(); + + Assert.Equal(exportedFiles.Select(file => file.Relative).ToArray(), filesElement.Select(file => file.Path).ToArray()); + + long totalBytes = exportedFiles.Select(file => new FileInfo(file.Absolute).Length).Sum(); + Assert.Equal(totalBytes, root.GetProperty("totalBytes").GetInt64()); + Assert.Equal(exportedFiles.Length, root.GetProperty("fileCount").GetInt32()); + + var digest = root.GetProperty("digest").GetString(); + var digestResult = new JsonExportResult( + exportDirectory, + exportedAt, + exportedFiles.Select(file => + { + var manifestEntry = filesElement.First(f => f.Path == file.Relative); + if (manifestEntry.Digest is null) + { + throw new InvalidOperationException($"Manifest entry for {file.Relative} missing digest."); + } + + return new JsonExportFile(file.Relative, new FileInfo(file.Absolute).Length, manifestEntry.Digest); + }), + exportedFiles.Length, + totalBytes); + var expectedDigest = ExportDigestCalculator.ComputeTreeDigest(digestResult); + Assert.Equal(expectedDigest, digest); + + var exporterVersion = root.GetProperty("exporterVersion").GetString(); + Assert.Equal(ExporterVersion.GetVersion(typeof(JsonFeedExporter)), exporterVersion); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + + private sealed class StubAdvisoryStore : IAdvisoryStore + { + private readonly IReadOnlyList _advisories; + + public StubAdvisoryStore(params Advisory[] advisories) + { + _advisories = advisories; + } + + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + => Task.FromResult(_advisories); + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + => Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + => Task.CompletedTask; + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) + { + return EnumerateAsync(cancellationToken); + + async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) + { + foreach (var advisory in _advisories) + { + ct.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + } + } + + private sealed class InMemoryExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; + + public Task FindAsync(string id, CancellationToken cancellationToken) + => Task.FromResult(_record); + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now; + + public TestTimeProvider(DateTimeOffset start) => _now = start; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan delta) => _now = _now.Add(delta); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj b/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj new file mode 100644 index 00000000..c8ac735f --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs new file mode 100644 index 00000000..34e57e28 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs @@ -0,0 +1,148 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json.Tests; + +public sealed class VulnListJsonExportPathResolverTests +{ + private static readonly DateTimeOffset DefaultPublished = DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture); + + [Fact] + public void ResolvesCvePath() + { + var advisory = CreateAdvisory("CVE-2024-1234"); + var resolver = new VulnListJsonExportPathResolver(); + + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("nvd", "2024", "CVE-2024-1234.json"), path); + } + + [Fact] + public void ResolvesGhsaWithPackage() + { + var package = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:go/github.com/acme/widget@1.0.0", + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: Array.Empty()); + + var advisory = CreateAdvisory( + "GHSA-aaaa-bbbb-cccc", + aliases: new[] { "CVE-2024-2000" }, + packages: new[] { package }); + var resolver = new VulnListJsonExportPathResolver(); + + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("ghsa", "go", "github.com%2Facme%2Fwidget", "GHSA-AAAA-BBBB-CCCC.json"), path); + } + + [Fact] + public void ResolvesUbuntuUsn() + { + var advisory = CreateAdvisory("USN-6620-1"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("ubuntu", "USN-6620-1.json"), path); + } + + [Fact] + public void ResolvesDebianDla() + { + var advisory = CreateAdvisory("DLA-1234-1"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("debian", "DLA-1234-1.json"), path); + } + + [Fact] + public void ResolvesRedHatRhsa() + { + var advisory = CreateAdvisory("RHSA-2024:0252"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("redhat", "oval", "RHSA-2024_0252.json"), path); + } + + [Fact] + public void ResolvesAmazonAlas() + { + var advisory = CreateAdvisory("ALAS2-2024-1234"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("amazon", "2", "ALAS2-2024-1234.json"), path); + } + + [Fact] + public void ResolvesOracleElsa() + { + var advisory = CreateAdvisory("ELSA-2024-12345"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("oracle", "linux", "ELSA-2024-12345.json"), path); + } + + [Fact] + public void ResolvesRockyRlsa() + { + var advisory = CreateAdvisory("RLSA-2024:0417"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("rocky", "RLSA-2024_0417.json"), path); + } + + [Fact] + public void ResolvesByProvenanceFallback() + { + var provenance = new[] { new AdvisoryProvenance("wolfi", "map", "", DefaultPublished) }; + var advisory = CreateAdvisory("WOLFI-2024-0001", provenance: provenance); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("wolfi", "WOLFI-2024-0001.json"), path); + } + + [Fact] + public void DefaultsToMiscWhenUnmapped() + { + var advisory = CreateAdvisory("CUSTOM-2024-99"); + var resolver = new VulnListJsonExportPathResolver(); + var path = resolver.GetRelativePath(advisory); + + Assert.Equal(Path.Combine("misc", "CUSTOM-2024-99.json"), path); + } + + private static Advisory CreateAdvisory( + string advisoryKey, + IEnumerable? aliases = null, + IEnumerable? packages = null, + IEnumerable? provenance = null) + { + return new Advisory( + advisoryKey: advisoryKey, + title: $"Advisory {advisoryKey}", + summary: null, + language: "en", + published: DefaultPublished, + modified: DefaultPublished, + severity: "medium", + exploitKnown: false, + aliases: aliases ?? Array.Empty(), + references: Array.Empty(), + affectedPackages: packages ?? Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: provenance ?? Array.Empty()); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/AGENTS.md b/src/StellaOps.Feedser.Exporter.Json/AGENTS.md new file mode 100644 index 00000000..80141ef9 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +Optional exporter producing vuln-list-shaped JSON tree for downstream trivy-db builder or interoperability. Deterministic, provenance-preserving. +## Scope +- Transform canonical advisories into directory tree structure mirroring aquasecurity/vuln-list (by ecosystem/vendor/distro as applicable). +- Sorting and serialization invariants: stable key order, newline policy, UTC ISO-8601. +- Cursoring/incremental export: export_state tracks last advisory hash/time to avoid full rewrites. +- Packaging: output directory under exports/json/ with reproducible naming; optionally symlink latest. +- Optional auxiliary index files (for example severity summaries) may be generated when explicitly requested, but must remain deterministic and avoid altering canonical payloads. +## Participants +- Storage.Mongo.AdvisoryStore as input; ExportState repository for cursors/digests. +- Core scheduler runs JsonExportJob; Plugin DI wires JsonExporter + job. +- TrivyDb exporter may consume the rendered tree in v0 (builder path) if configured. +## Interfaces & contracts +- Job kind: export:json (JsonExportJob). +- Determinism: same inputs -> identical file bytes; hash snapshot persisted. +- Provenance: include minimal provenance fields when helpful; keep identity stable. +## In/Out of scope +In: JSON rendering and layout; incremental/deterministic writes. +Out: ORAS push and Trivy DB BoltDB writing (owned by Trivy exporter). +## Observability & security expectations +- Metrics: export.json.records, bytes, duration, delta.changed. +- Logs: target path, record counts, digest; no sensitive data. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Exporter.Json.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs b/src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs new file mode 100644 index 00000000..1e386765 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Feedser.Exporter.Json; + +public static class ExportDigestCalculator +{ + public static string ComputeTreeDigest(JsonExportResult result) + { + ArgumentNullException.ThrowIfNull(result); + + using var sha256 = SHA256.Create(); + var buffer = new byte[128 * 1024]; + + foreach (var file in result.FilePaths.OrderBy(static path => path, StringComparer.Ordinal)) + { + var normalized = file.Replace("\\", "/"); + var pathBytes = Encoding.UTF8.GetBytes(normalized); + _ = sha256.TransformBlock(pathBytes, 0, pathBytes.Length, null, 0); + + var fullPath = ResolveFullPath(result.ExportDirectory, normalized); + using var stream = File.OpenRead(fullPath); + int read; + while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) + { + _ = sha256.TransformBlock(buffer, 0, read, null, 0); + } + } + + _ = sha256.TransformFinalBlock(Array.Empty(), 0, 0); + var hash = sha256.Hash ?? Array.Empty(); + var hex = Convert.ToHexString(hash).ToLowerInvariant(); + return $"sha256:{hex}"; + } + + private static string ResolveFullPath(string root, string normalizedRelativePath) + { + var segments = normalizedRelativePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + var parts = new string[segments.Length + 1]; + parts[0] = root; + for (var i = 0; i < segments.Length; i++) + { + parts[i + 1] = segments[i]; + } + + return Path.Combine(parts); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs b/src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs new file mode 100644 index 00000000..aec83d1e --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs @@ -0,0 +1,28 @@ +using System; +using System.Reflection; + +namespace StellaOps.Feedser.Exporter.Json; + +public static class ExporterVersion +{ + public static string GetVersion(Type anchor) + { + ArgumentNullException.ThrowIfNull(anchor); + var assembly = anchor.Assembly; + + var informational = assembly.GetCustomAttribute()?.InformationalVersion; + if (!string.IsNullOrWhiteSpace(informational)) + { + return informational; + } + + var fileVersion = assembly.GetCustomAttribute()?.Version; + if (!string.IsNullOrWhiteSpace(fileVersion)) + { + return fileVersion!; + } + + var version = assembly.GetName().Version; + return version?.ToString() ?? "0.0.0"; + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs b/src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs new file mode 100644 index 00000000..a14314a6 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs @@ -0,0 +1,12 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json; + +public interface IJsonExportPathResolver +{ + /// + /// Returns the relative path (using platform directory separators) for the supplied advisory. + /// Path must not include the leading export root. + /// + string GetRelativePath(Advisory advisory); +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs new file mode 100644 index 00000000..0cbf1204 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs @@ -0,0 +1,37 @@ +using System; + +namespace StellaOps.Feedser.Exporter.Json; + +/// +/// Metadata describing a single file produced by the JSON exporter. +/// +public sealed class JsonExportFile +{ + public JsonExportFile(string relativePath, long length, string digest) + { + RelativePath = relativePath ?? throw new ArgumentNullException(nameof(relativePath)); + if (relativePath.Length == 0) + { + throw new ArgumentException("Relative path cannot be empty.", nameof(relativePath)); + } + + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + Digest = digest ?? throw new ArgumentNullException(nameof(digest)); + if (digest.Length == 0) + { + throw new ArgumentException("Digest cannot be empty.", nameof(digest)); + } + + Length = length; + } + + public string RelativePath { get; } + + public long Length { get; } + + public string Digest { get; } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs new file mode 100644 index 00000000..a6a1a01a --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs @@ -0,0 +1,30 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonExportJob : IJob +{ + public const string JobKind = "export:json"; + public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(10); + public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(5); + + private readonly JsonFeedExporter _exporter; + private readonly ILogger _logger; + + public JsonExportJob(JsonFeedExporter exporter, ILogger logger) + { + _exporter = exporter ?? throw new ArgumentNullException(nameof(exporter)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + _logger.LogInformation("Executing JSON export job {RunId}", context.RunId); + await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Completed JSON export job {RunId}", context.RunId); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs new file mode 100644 index 00000000..ef8f6c89 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Exporter.Json; + +internal static class JsonExportManifestWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true, + }; + + public static async Task WriteAsync( + JsonExportResult result, + string digest, + string exporterVersion, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(result); + ArgumentException.ThrowIfNullOrEmpty(digest); + ArgumentException.ThrowIfNullOrEmpty(exporterVersion); + + var exportId = Path.GetFileName(result.ExportDirectory); + var files = result.Files + .Select(static file => new JsonExportManifestFile(file.RelativePath.Replace("\\", "/", StringComparison.Ordinal), file.Length, file.Digest)) + .ToArray(); + + var manifest = new JsonExportManifest( + exportId, + result.ExportedAt.UtcDateTime, + digest, + result.AdvisoryCount, + result.TotalBytes, + files.Length, + files, + exporterVersion); + + var payload = JsonSerializer.SerializeToUtf8Bytes(manifest, SerializerOptions); + var manifestPath = Path.Combine(result.ExportDirectory, "manifest.json"); + await File.WriteAllBytesAsync(manifestPath, payload, cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(manifestPath, result.ExportedAt.UtcDateTime); + } + + private sealed record JsonExportManifest( + [property: JsonPropertyOrder(1)] string ExportId, + [property: JsonPropertyOrder(2)] DateTime GeneratedAt, + [property: JsonPropertyOrder(3)] string Digest, + [property: JsonPropertyOrder(4)] int AdvisoryCount, + [property: JsonPropertyOrder(5)] long TotalBytes, + [property: JsonPropertyOrder(6)] int FileCount, + [property: JsonPropertyOrder(7)] IReadOnlyList Files, + [property: JsonPropertyOrder(8)] string ExporterVersion); + + private sealed record JsonExportManifestFile( + [property: JsonPropertyOrder(1)] string Path, + [property: JsonPropertyOrder(2)] long Bytes, + [property: JsonPropertyOrder(3)] string Digest); +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs new file mode 100644 index 00000000..ec5d77d7 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs @@ -0,0 +1,34 @@ +using System.IO; + +namespace StellaOps.Feedser.Exporter.Json; + +/// +/// Configuration for JSON exporter output paths and determinism controls. +/// +public sealed class JsonExportOptions +{ + /// + /// Root directory where exports are written. Default "exports/json". + /// + public string OutputRoot { get; set; } = Path.Combine("exports", "json"); + + /// + /// Format string applied to the export timestamp to produce the directory name. + /// + public string DirectoryNameFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; + + /// + /// Optional static name for the symlink (or directory junction) pointing at the most recent export. + /// + public string LatestSymlinkName { get; set; } = "latest"; + + /// + /// When true, attempts to re-point after a successful export. + /// + public bool MaintainLatestSymlink { get; set; } = true; + + /// + /// Optional repository identifier recorded alongside export state metadata. + /// + public string? TargetRepository { get; set; } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs new file mode 100644 index 00000000..8d4a35ef --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonExportResult +{ + public JsonExportResult( + string exportDirectory, + DateTimeOffset exportedAt, + IEnumerable files, + int advisoryCount, + long totalBytes) + { + if (string.IsNullOrWhiteSpace(exportDirectory)) + { + throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory)); + } + + ExportDirectory = exportDirectory; + ExportedAt = exportedAt; + AdvisoryCount = advisoryCount; + TotalBytes = totalBytes; + + var list = (files ?? throw new ArgumentNullException(nameof(files))) + .Where(static file => file is not null) + .ToImmutableArray(); + + Files = list; + FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray(); + } + + public string ExportDirectory { get; } + + public DateTimeOffset ExportedAt { get; } + + public ImmutableArray Files { get; } + + public ImmutableArray FilePaths { get; } + + public int AdvisoryCount { get; } + + public long TotalBytes { get; } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs new file mode 100644 index 00000000..637d75af --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs @@ -0,0 +1,239 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Exporter.Json; + +/// +/// Writes canonical advisory snapshots into a vuln-list style directory tree with deterministic ordering. +/// +public sealed class JsonExportSnapshotBuilder +{ + private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + private readonly JsonExportOptions _options; + private readonly IJsonExportPathResolver _pathResolver; + + public JsonExportSnapshotBuilder(JsonExportOptions options, IJsonExportPathResolver pathResolver) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); + } + + public Task WriteAsync( + IReadOnlyCollection advisories, + DateTimeOffset exportedAt, + string? exportName = null, + CancellationToken cancellationToken = default) + { + if (advisories is null) + { + throw new ArgumentNullException(nameof(advisories)); + } + + return WriteAsync(EnumerateAsync(advisories, cancellationToken), exportedAt, exportName, cancellationToken); + } + + public async Task WriteAsync( + IAsyncEnumerable advisories, + DateTimeOffset exportedAt, + string? exportName = null, + CancellationToken cancellationToken = default) + { + if (advisories is null) + { + throw new ArgumentNullException(nameof(advisories)); + } + + var exportDirectoryName = exportName ?? exportedAt.UtcDateTime.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture); + if (string.IsNullOrWhiteSpace(exportDirectoryName)) + { + throw new InvalidOperationException("Export directory name resolved to an empty string."); + } + + var exportRoot = EnsureDirectoryExists(Path.GetFullPath(_options.OutputRoot)); + TrySetDirectoryTimestamp(exportRoot, exportedAt); + var exportDirectory = Path.Combine(exportRoot, exportDirectoryName); + + if (Directory.Exists(exportDirectory)) + { + Directory.Delete(exportDirectory, recursive: true); + } + + Directory.CreateDirectory(exportDirectory); + TrySetDirectoryTimestamp(exportDirectory, exportedAt); + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var files = new List(); + long totalBytes = 0L; + var advisoryCount = 0; + + await foreach (var advisory in advisories.WithCancellation(cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + + advisoryCount++; + var entry = Resolve(advisory); + if (!seen.Add(entry.RelativePath)) + { + throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'."); + } + + var destination = Combine(exportDirectory, entry.Segments); + var destinationDirectory = Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(destinationDirectory)) + { + EnsureDirectoryExists(destinationDirectory); + TrySetDirectoryTimestamp(destinationDirectory, exportedAt); + } + var payload = SnapshotSerializer.ToSnapshot(entry.Advisory); + var bytes = Utf8NoBom.GetBytes(payload); + + await File.WriteAllBytesAsync(destination, bytes, cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(destination, exportedAt.UtcDateTime); + + var digest = ComputeDigest(bytes); + files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest)); + totalBytes += bytes.LongLength; + } + + files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath)); + + return new JsonExportResult(exportDirectory, exportedAt, files, advisoryCount, totalBytes); + } + + private static async IAsyncEnumerable EnumerateAsync( + IEnumerable advisories, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + foreach (var advisory in advisories) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + + private static string EnsureDirectoryExists(string directory) + { + if (string.IsNullOrWhiteSpace(directory)) + { + throw new ArgumentException("Directory path must be provided.", nameof(directory)); + } + + Directory.CreateDirectory(directory); + return directory; + } + + private static string Combine(string root, IReadOnlyList segments) + { + var parts = new string[segments.Count + 1]; + parts[0] = root; + for (var i = 0; i < segments.Count; i++) + { + parts[i + 1] = segments[i]; + } + + return Path.Combine(parts); + } + + private static void TrySetDirectoryTimestamp(string directory, DateTimeOffset timestamp) + { + try + { + Directory.SetLastWriteTimeUtc(directory, timestamp.UtcDateTime); + } + catch (IOException) + { + // Ignore failure to set timestamps; not critical for content determinism. + } + catch (UnauthorizedAccessException) + { + // Ignore permission issues when setting timestamps. + } + catch (PlatformNotSupportedException) + { + // Some platforms may not support this operation. + } + } + + private PathResolution Resolve(Advisory advisory) + { + if (advisory is null) + { + throw new ArgumentNullException(nameof(advisory)); + } + + var relativePath = _pathResolver.GetRelativePath(advisory); + var segments = NormalizeRelativePath(relativePath); + var normalized = string.Join('/', segments); + return new PathResolution(advisory, normalized, segments); + } + + private static string[] NormalizeRelativePath(string relativePath) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + throw new InvalidOperationException("Path resolver returned an empty path."); + } + + if (Path.IsPathRooted(relativePath)) + { + throw new InvalidOperationException("Path resolver returned an absolute path; only relative paths are supported."); + } + + var pieces = relativePath.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); + if (pieces.Length == 0) + { + throw new InvalidOperationException("Path resolver produced no path segments."); + } + + var sanitized = new string[pieces.Length]; + for (var i = 0; i < pieces.Length; i++) + { + var segment = pieces[i]; + if (segment == "." || segment == "..") + { + throw new InvalidOperationException("Relative paths cannot include '.' or '..' segments."); + } + + sanitized[i] = SanitizeSegment(segment); + } + + return sanitized; + } + + private static string SanitizeSegment(string segment) + { + var invalid = Path.GetInvalidFileNameChars(); + Span buffer = stackalloc char[segment.Length]; + var count = 0; + foreach (var ch in segment) + { + if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0) + { + buffer[count++] = '_'; + } + else + { + buffer[count++] = ch; + } + } + + var sanitized = new string(buffer[..count]).Trim(); + return string.IsNullOrEmpty(sanitized) ? "_" : sanitized; + } + + private sealed record PathResolution(Advisory Advisory, string RelativePath, IReadOnlyList Segments); + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash).ToLowerInvariant(); + return $"sha256:{hex}"; + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs new file mode 100644 index 00000000..da8717ac --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs @@ -0,0 +1,59 @@ +using System; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:exporters:json"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Bind(configuration.GetSection(ConfigurationSection)) + .PostConfigure(static options => + { + if (string.IsNullOrWhiteSpace(options.OutputRoot)) + { + options.OutputRoot = Path.Combine("exports", "json"); + } + + if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat)) + { + options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'"; + } + }); + + services.AddSingleton(); + services.AddTransient(); + + services.PostConfigure(options => + { + if (!options.Definitions.ContainsKey(JsonExportJob.JobKind)) + { + options.Definitions[JsonExportJob.JobKind] = new JobDefinition( + JsonExportJob.JobKind, + typeof(JsonExportJob), + JsonExportJob.DefaultTimeout, + JsonExportJob.DefaultLeaseDuration, + null, + true); + } + }); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs new file mode 100644 index 00000000..d0e74059 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs @@ -0,0 +1,23 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonExporterPlugin : IExporterPlugin +{ + public string Name => JsonFeedExporter.ExporterName; + + public bool IsAvailable(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetService() is not null; + } + + public IFeedExporter Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs b/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs new file mode 100644 index 00000000..56bf1cc6 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs @@ -0,0 +1,150 @@ +using System; +using System.Globalization; +using System.IO; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Exporter.Json; + +public sealed class JsonFeedExporter : IFeedExporter +{ + public const string ExporterName = "json"; + public const string ExporterId = "export:json"; + + private readonly IAdvisoryStore _advisoryStore; + private readonly JsonExportOptions _options; + private readonly IJsonExportPathResolver _pathResolver; + private readonly ExportStateManager _stateManager; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _exporterVersion; + + public JsonFeedExporter( + IAdvisoryStore advisoryStore, + IOptions options, + IJsonExportPathResolver pathResolver, + ExportStateManager stateManager, + ILogger logger, + TimeProvider? timeProvider = null) + { + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); + _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter)); + } + + public string Name => ExporterName; + + public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var exportedAt = _timeProvider.GetUtcNow(); + var exportId = exportedAt.ToString(_options.DirectoryNameFormat, CultureInfo.InvariantCulture); + var exportRoot = Path.GetFullPath(_options.OutputRoot); + + _logger.LogInformation("Starting JSON export {ExportId}", exportId); + + var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); + + var builder = new JsonExportSnapshotBuilder(_options, _pathResolver); + var advisoryStream = _advisoryStore.StreamAsync(cancellationToken); + var result = await builder.WriteAsync(advisoryStream, exportedAt, exportId, cancellationToken).ConfigureAwait(false); + + var digest = ExportDigestCalculator.ComputeTreeDigest(result); + _logger.LogInformation( + "JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}", + exportId, + result.Files.Length, + result.TotalBytes, + result.AdvisoryCount, + digest); + + if (existingState is not null && string.Equals(existingState.LastFullDigest, digest, StringComparison.Ordinal)) + { + _logger.LogInformation("JSON export {ExportId} produced unchanged digest; skipping state update.", exportId); + TryDeleteDirectory(result.ExportDirectory); + return; + } + + await _stateManager.StoreFullExportAsync( + ExporterId, + exportId, + digest, + cursor: digest, + targetRepository: _options.TargetRepository, + exporterVersion: _exporterVersion, + cancellationToken: cancellationToken).ConfigureAwait(false); + + await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false); + + if (_options.MaintainLatestSymlink) + { + TryUpdateLatestSymlink(exportRoot, result.ExportDirectory); + } + } + + private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory) + { + if (string.IsNullOrWhiteSpace(_options.LatestSymlinkName)) + { + return; + } + + var latestPath = Path.Combine(exportRoot, _options.LatestSymlinkName); + + try + { + if (Directory.Exists(latestPath) || File.Exists(latestPath)) + { + TryRemoveExistingPointer(latestPath); + } + + Directory.CreateSymbolicLink(latestPath, exportDirectory); + _logger.LogDebug("Updated latest JSON export pointer to {Target}", exportDirectory); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException) + { + _logger.LogWarning(ex, "Failed to update latest JSON export pointer at {LatestPath}", latestPath); + } + } + + private void TryRemoveExistingPointer(string latestPath) + { + try + { + var attributes = File.GetAttributes(latestPath); + if (attributes.HasFlag(FileAttributes.Directory)) + { + Directory.Delete(latestPath, recursive: false); + } + else + { + File.Delete(latestPath); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to remove existing latest pointer {LatestPath}", latestPath); + } + } + + private void TryDeleteDirectory(string path) + { + try + { + if (Directory.Exists(path)) + { + Directory.Delete(path, recursive: true); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to remove unchanged export directory {ExportDirectory}", path); + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj b/src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj new file mode 100644 index 00000000..565cd3dd --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Exporter.Json/TASKS.md b/src/StellaOps.Feedser.Exporter.Json/TASKS.md new file mode 100644 index 00000000..9f726d04 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Directory layout strategy (vuln-list mirror)|BE-Export|Models|DONE – `VulnListJsonExportPathResolver` maps CVE, GHSA, distro, and vendor identifiers into vuln-list style paths.| +|Deterministic serializer|BE-Export|Models|DONE – Canonical serializer + snapshot builder emit stable JSON across runs.| +|ExportState read/write|BE-Export|Storage.Mongo|DONE – `JsonFeedExporter` reads prior state, stores digests/cursors, and skips unchanged exports.| +|JsonExportJob wiring|BE-Export|Core|DONE – Job scheduler options now configurable via DI; JSON job registered with scheduler.| +|Snapshot tests for file tree|QA|Exporters|DONE – Added resolver/exporter tests asserting tree layout and deterministic behavior.| +|Parity smoke vs upstream vuln-list|QA|Exporters|DONE – `JsonExporterParitySmokeTests` covers common ecosystems against vuln-list layout.| +|Stream advisories during export|BE-Export|Storage.Mongo|DONE – exporter + streaming-only test ensures single enumeration and per-file digest capture.| +|Emit export manifest with digest metadata|BE-Export|Exporters|DONE – manifest now includes per-file digests/sizes alongside tree digest.| diff --git a/src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs b/src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs new file mode 100644 index 00000000..a34018fd --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs @@ -0,0 +1,455 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Identifiers; + +namespace StellaOps.Feedser.Exporter.Json; + +/// +/// Path resolver approximating the directory layout used by aquasecurity/vuln-list. +/// Handles common vendor, distro, and ecosystem shapes with deterministic fallbacks. +/// +public sealed class VulnListJsonExportPathResolver : IJsonExportPathResolver +{ + private static readonly Regex CvePattern = new("^CVE-(?\\d{4})-(?\\d{4,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex GhsaPattern = new("^GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex UsnPattern = new("^USN-(?\\d+-\\d+)(?[a-z])?$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex DebianPattern = new("^(?DLA|DSA|ELA)-(?\\d+-\\d+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex RedHatPattern = new("^RH(?SA|BA|EA)-(?[0-9:.-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex AmazonPattern = new("^ALAS(?2|2022|2023)?-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex OraclePattern = new("^(?ELSA|ELBA|ELSA-OCI|ELBA-OCI)-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex PhotonPattern = new("^PHSA-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex RockyPattern = new("^RLSA-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex SusePattern = new("^SUSE-(?SU|RU|OU|SB)-(?[0-9A-Za-z:._-]+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + + private static readonly Dictionary SourceDirectoryMap = new(StringComparer.OrdinalIgnoreCase) + { + ["nvd"] = new[] { "nvd" }, + ["ghsa"] = new[] { "ghsa" }, + ["github"] = new[] { "ghsa" }, + ["osv"] = new[] { "osv" }, + ["redhat"] = new[] { "redhat", "oval" }, + ["ubuntu"] = new[] { "ubuntu" }, + ["debian"] = new[] { "debian" }, + ["oracle"] = new[] { "oracle" }, + ["photon"] = new[] { "photon" }, + ["rocky"] = new[] { "rocky" }, + ["suse"] = new[] { "suse" }, + ["amazon"] = new[] { "amazon" }, + ["aws"] = new[] { "amazon" }, + ["alpine"] = new[] { "alpine" }, + ["wolfi"] = new[] { "wolfi" }, + ["chainguard"] = new[] { "chainguard" }, + ["cert-fr"] = new[] { "cert", "fr" }, + ["cert-in"] = new[] { "cert", "in" }, + ["cert-cc"] = new[] { "cert", "cc" }, + ["cert-bund"] = new[] { "cert", "bund" }, + ["cisa"] = new[] { "ics", "cisa" }, + ["ics-cisa"] = new[] { "ics", "cisa" }, + ["ics-kaspersky"] = new[] { "ics", "kaspersky" }, + ["kaspersky"] = new[] { "ics", "kaspersky" }, + }; + + private static readonly Dictionary GhsaEcosystemMap = new(StringComparer.OrdinalIgnoreCase) + { + ["go"] = "go", + ["golang"] = "go", + ["npm"] = "npm", + ["maven"] = "maven", + ["pypi"] = "pip", + ["pip"] = "pip", + ["nuget"] = "nuget", + ["composer"] = "composer", + ["packagist"] = "composer", + ["rubygems"] = "rubygems", + ["gem"] = "rubygems", + ["swift"] = "swift", + ["cargo"] = "cargo", + ["hex"] = "hex", + ["pub"] = "pub", + ["github"] = "github", + ["docker"] = "container", + }; + + public string GetRelativePath(Advisory advisory) + { + if (advisory is null) + { + throw new ArgumentNullException(nameof(advisory)); + } + + var identifier = SelectPreferredIdentifier(advisory); + if (identifier.Length == 0) + { + throw new InvalidOperationException("Unable to derive identifier for advisory."); + } + + var layout = ResolveLayout(advisory, identifier); + var segments = new string[layout.Segments.Length + 1]; + for (var i = 0; i < layout.Segments.Length; i++) + { + segments[i] = layout.Segments[i]; + } + segments[^1] = layout.FileName; + return Path.Combine(segments); + } + + private static Layout ResolveLayout(Advisory advisory, string identifier) + { + if (TryResolveCve(identifier, out var layout)) + { + return layout; + } + + if (TryResolveGhsa(advisory, identifier, out layout)) + { + return layout; + } + + if (TryResolveUsn(identifier, out layout) || + TryResolveDebian(identifier, out layout) || + TryResolveRedHat(identifier, out layout) || + TryResolveAmazon(identifier, out layout) || + TryResolveOracle(identifier, out layout) || + TryResolvePhoton(identifier, out layout) || + TryResolveRocky(identifier, out layout) || + TryResolveSuse(identifier, out layout)) + { + return layout; + } + + if (TryResolveByProvenance(advisory, identifier, out layout)) + { + return layout; + } + + return new Layout(new[] { "misc" }, CreateFileName(identifier)); + } + + private static bool TryResolveCve(string identifier, out Layout layout) + { + var match = CvePattern.Match(identifier); + if (!match.Success) + { + layout = default; + return false; + } + + var year = match.Groups["year"].Value; + layout = new Layout(new[] { "nvd", year }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveGhsa(Advisory advisory, string identifier, out Layout layout) + { + if (!GhsaPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + if (TryGetGhsaPackage(advisory, out var ecosystem, out var packagePath)) + { + layout = new Layout(new[] { "ghsa", ecosystem, packagePath }, CreateFileName(identifier, uppercase: true)); + return true; + } + + layout = new Layout(new[] { "github", "advisories" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveUsn(string identifier, out Layout layout) + { + if (!UsnPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "ubuntu" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveDebian(string identifier, out Layout layout) + { + var match = DebianPattern.Match(identifier); + if (!match.Success) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "debian" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveRedHat(string identifier, out Layout layout) + { + if (!RedHatPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "redhat", "oval" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveAmazon(string identifier, out Layout layout) + { + var match = AmazonPattern.Match(identifier); + if (!match.Success) + { + layout = default; + return false; + } + + var channel = match.Groups["channel"].Value; + var subdirectory = channel switch + { + "2" => "2", + "2023" => "2023", + "2022" => "2022", + _ => "1", + }; + + layout = new Layout(new[] { "amazon", subdirectory }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveOracle(string identifier, out Layout layout) + { + if (!OraclePattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "oracle", "linux" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolvePhoton(string identifier, out Layout layout) + { + if (!PhotonPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "photon" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveRocky(string identifier, out Layout layout) + { + if (!RockyPattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "rocky" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveSuse(string identifier, out Layout layout) + { + if (!SusePattern.IsMatch(identifier)) + { + layout = default; + return false; + } + + layout = new Layout(new[] { "suse" }, CreateFileName(identifier, uppercase: true)); + return true; + } + + private static bool TryResolveByProvenance(Advisory advisory, string identifier, out Layout layout) + { + foreach (var source in EnumerateDistinctProvenanceSources(advisory)) + { + if (SourceDirectoryMap.TryGetValue(source, out var segments)) + { + layout = new Layout(segments, CreateFileName(identifier)); + return true; + } + } + + layout = default; + return false; + } + + private static bool TryGetGhsaPackage(Advisory advisory, out string ecosystem, out string packagePath) + { + foreach (var package in advisory.AffectedPackages) + { + if (!TryParsePackageUrl(package.Identifier, out var type, out var encodedPath)) + { + continue; + } + + if (GhsaEcosystemMap.TryGetValue(type, out var mapped)) + { + ecosystem = mapped; + } + else + { + ecosystem = type.ToLowerInvariant(); + } + + packagePath = encodedPath; + return true; + } + + ecosystem = "advisories"; + packagePath = "_"; + return false; + } + + private static bool TryParsePackageUrl(string identifier, out string type, out string encodedPath) + { + type = string.Empty; + encodedPath = string.Empty; + + if (!IdentifierNormalizer.TryNormalizePackageUrl(identifier, out _, out var packageUrl)) + { + return false; + } + + var segments = packageUrl!.NamespaceSegments.IsDefaultOrEmpty + ? new[] { packageUrl.Name } + : packageUrl.NamespaceSegments.Append(packageUrl.Name).ToArray(); + + type = packageUrl.Type; + encodedPath = string.Join("%2F", segments); + return true; + } + + private static string CreateFileName(string identifier, bool uppercase = false) + { + var candidate = uppercase ? identifier.ToUpperInvariant() : identifier; + return $"{SanitizeFileName(candidate)}.json"; + } + + private static IEnumerable EnumerateDistinctProvenanceSources(Advisory advisory) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var source in advisory.Provenance) + { + if (TryAddSource(source.Source)) + { + yield return source.Source; + } + } + + foreach (var reference in advisory.References) + { + if (TryAddSource(reference.Provenance.Source)) + { + yield return reference.Provenance.Source; + } + } + + foreach (var package in advisory.AffectedPackages) + { + foreach (var source in package.Provenance) + { + if (TryAddSource(source.Source)) + { + yield return source.Source; + } + } + + foreach (var range in package.VersionRanges) + { + if (TryAddSource(range.Provenance.Source)) + { + yield return range.Provenance.Source; + } + } + } + + foreach (var metric in advisory.CvssMetrics) + { + if (TryAddSource(metric.Provenance.Source)) + { + yield return metric.Provenance.Source; + } + } + + bool TryAddSource(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + return seen.Add(value); + } + } + + private static string SelectPreferredIdentifier(Advisory advisory) + { + if (TrySelectIdentifier(advisory.AdvisoryKey, out var preferred)) + { + return preferred; + } + + foreach (var alias in advisory.Aliases) + { + if (TrySelectIdentifier(alias, out preferred)) + { + return preferred; + } + } + + return advisory.AdvisoryKey.Trim(); + } + + private static bool TrySelectIdentifier(string value, out string identifier) + { + identifier = string.Empty; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (CvePattern.IsMatch(trimmed) || GhsaPattern.IsMatch(trimmed)) + { + identifier = trimmed; + return true; + } + + identifier = trimmed; + return false; + } + + private static string SanitizeFileName(string name) + { + var invalid = Path.GetInvalidFileNameChars(); + Span buffer = stackalloc char[name.Length]; + var count = 0; + foreach (var ch in name) + { + if (ch == '/' || ch == '\\' || Array.IndexOf(invalid, ch) >= 0) + { + buffer[count++] = '_'; + } + else + { + buffer[count++] = ch; + } + } + + var sanitized = new string(buffer[..count]).Trim(); + return string.IsNullOrEmpty(sanitized) ? "advisory" : sanitized; + } + + private readonly record struct Layout(string[] Segments, string FileName); +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj new file mode 100644 index 00000000..6ac1cb21 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs new file mode 100644 index 00000000..59314c02 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs @@ -0,0 +1,66 @@ +using System; +using StellaOps.Feedser.Exporter.TrivyDb; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbExportPlannerTests +{ + [Fact] + public void CreatePlan_ReturnsFullWhenStateMissing() + { + var planner = new TrivyDbExportPlanner(); + var plan = planner.CreatePlan(existingState: null, treeDigest: "sha256:abcd"); + + Assert.Equal(TrivyDbExportMode.Full, plan.Mode); + Assert.Equal("sha256:abcd", plan.TreeDigest); + Assert.Null(plan.BaseExportId); + Assert.Null(plan.BaseManifestDigest); + } + + [Fact] + public void CreatePlan_ReturnsSkipWhenCursorMatches() + { + var planner = new TrivyDbExportPlanner(); + var state = new ExportStateRecord( + Id: TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240810T000000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: null, + ExportCursor: "sha256:unchanged", + TargetRepository: "feedser/trivy", + ExporterVersion: "1.0", + UpdatedAt: DateTimeOffset.UtcNow); + + var plan = planner.CreatePlan(state, "sha256:unchanged"); + + Assert.Equal(TrivyDbExportMode.Skip, plan.Mode); + Assert.Equal("sha256:unchanged", plan.TreeDigest); + Assert.Equal("20240810T000000Z", plan.BaseExportId); + Assert.Equal("sha256:base", plan.BaseManifestDigest); + } + + [Fact] + public void CreatePlan_ReturnsFullWhenCursorDiffers() + { + var planner = new TrivyDbExportPlanner(); + var state = new ExportStateRecord( + Id: TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240810T000000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: null, + ExportCursor: "sha256:old", + TargetRepository: "feedser/trivy", + ExporterVersion: "1.0", + UpdatedAt: DateTimeOffset.UtcNow); + + var plan = planner.CreatePlan(state, "sha256:new"); + + Assert.Equal(TrivyDbExportMode.Full, plan.Mode); + Assert.Equal("sha256:new", plan.TreeDigest); + Assert.Equal("20240810T000000Z", plan.BaseExportId); + Assert.Equal("sha256:base", plan.BaseManifestDigest); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs new file mode 100644 index 00000000..5bf0aa61 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs @@ -0,0 +1,589 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Exporter.TrivyDb; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbFeedExporterTests : IDisposable +{ + private readonly string _root; + private readonly string _jsonRoot; + + public TrivyDbFeedExporterTests() + { + _root = Directory.CreateTempSubdirectory("feedser-trivy-exporter-tests").FullName; + _jsonRoot = Path.Combine(_root, "tree"); + } + + [Fact] + public async Task ExportAsync_SortsAdvisoriesByKeyDeterministically() + { + var advisoryB = CreateSampleAdvisory("CVE-2024-1002", "Second advisory"); + var advisoryA = CreateSampleAdvisory("CVE-2024-1001", "First advisory"); + + var advisoryStore = new StubAdvisoryStore(advisoryB, advisoryA); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + KeepWorkingTree = false, + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-20T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-21T00:00:00Z", + UpdatedAt = "2024-09-20T00:00:00Z", + }); + + var recordingBuilder = new RecordingTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + recordingBuilder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var paths = recordingBuilder.LastRelativePaths; + Assert.NotNull(paths); + + var sorted = paths!.OrderBy(static p => p, StringComparer.Ordinal).ToArray(); + Assert.Equal(sorted, paths); + + advisoryStore.SetAdvisories(advisoryA, advisoryB); + timeProvider.Advance(TimeSpan.FromMinutes(7)); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal("20240920T000000Z", record!.BaseExportId); + Assert.Single(recordingBuilder.ManifestDigests); + } + + [Fact] + public async Task ExportAsync_SmallDatasetProducesDeterministicOciLayout() + { + var advisories = new[] + { + CreateSampleAdvisory("CVE-2024-3000", "Demo advisory 1"), + CreateSampleAdvisory("CVE-2024-3001", "Demo advisory 2"), + }; + + var run1 = await RunDeterministicExportAsync(advisories); + var run2 = await RunDeterministicExportAsync(advisories); + + Assert.Equal(run1.ManifestDigest, run2.ManifestDigest); + Assert.Equal(run1.IndexJson, run2.IndexJson); + Assert.Equal(run1.MetadataJson, run2.MetadataJson); + Assert.Equal(run1.ManifestJson, run2.ManifestJson); + + var digests1 = run1.Blobs.Keys.OrderBy(static d => d, StringComparer.Ordinal).ToArray(); + var digests2 = run2.Blobs.Keys.OrderBy(static d => d, StringComparer.Ordinal).ToArray(); + Assert.Equal(digests1, digests2); + + foreach (var digest in digests1) + { + Assert.True(run2.Blobs.TryGetValue(digest, out var other), $"Missing digest {digest} in second run"); + Assert.True(run1.Blobs[digest].SequenceEqual(other), $"Blob {digest} differs between runs"); + } + + using var metadataDoc = JsonDocument.Parse(run1.MetadataJson); + Assert.Equal(2, metadataDoc.RootElement.GetProperty("advisoryCount").GetInt32()); + + using var manifestDoc = JsonDocument.Parse(run1.ManifestJson); + Assert.Equal(TrivyDbMediaTypes.TrivyConfig, manifestDoc.RootElement.GetProperty("config").GetProperty("mediaType").GetString()); + var layer = manifestDoc.RootElement.GetProperty("layers")[0]; + Assert.Equal(TrivyDbMediaTypes.TrivyLayer, layer.GetProperty("mediaType").GetString()); + } + + [Fact] + public void ExportOptions_GetExportRoot_NormalizesRelativeRoot() + { + var options = new TrivyDbExportOptions + { + OutputRoot = Path.Combine("..", "exports", "trivy-test"), + }; + + var exportId = "20240901T000000Z"; + var path = options.GetExportRoot(exportId); + + Assert.True(Path.IsPathRooted(path)); + Assert.EndsWith(Path.Combine("exports", "trivy-test", exportId), path, StringComparison.Ordinal); + } + + [Fact] + public async Task ExportAsync_PersistsStateAndSkipsWhenDigestUnchanged() + { + var advisory = CreateSampleAdvisory(); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-01T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-02T00:00:00Z", + UpdatedAt = "2024-09-01T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal("20240901T000000Z", record!.BaseExportId); + Assert.False(string.IsNullOrEmpty(record.ExportCursor)); + + var baseExportId = record.BaseExportId ?? string.Empty; + Assert.False(string.IsNullOrEmpty(baseExportId)); + var firstExportDirectory = Path.Combine(_root, baseExportId); + Assert.True(Directory.Exists(firstExportDirectory)); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + await exporter.ExportAsync(provider, CancellationToken.None); + + var updatedRecord = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(updatedRecord); + Assert.Equal(record.UpdatedAt, updatedRecord!.UpdatedAt); + Assert.Equal(record.LastFullDigest, updatedRecord.LastFullDigest); + + var skippedExportDirectory = Path.Combine(_root, "20240901T000500Z"); + Assert.False(Directory.Exists(skippedExportDirectory)); + + Assert.Empty(orasPusher.Pushes); + } + + [Fact] + public async Task ExportAsync_CreatesOfflineBundle() + { + var advisory = CreateSampleAdvisory(); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + OfflineBundle = new TrivyDbOfflineBundleOptions + { + Enabled = true, + FileName = "{exportId}.bundle.tar.gz", + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-15T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-16T00:00:00Z", + UpdatedAt = "2024-09-15T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = "20240915T000000Z"; + var bundlePath = Path.Combine(_root, $"{exportId}.bundle.tar.gz"); + Assert.True(File.Exists(bundlePath)); + Assert.Empty(orasPusher.Pushes); + } + + private static Advisory CreateSampleAdvisory( + string advisoryKey = "CVE-2024-9999", + string title = "Trivy Export Test") + { + return new Advisory( + advisoryKey: advisoryKey, + title: title, + summary: null, + language: "en", + published: DateTimeOffset.Parse("2024-08-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-08-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2024-9999" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + + private sealed class StubAdvisoryStore : IAdvisoryStore + { + private IReadOnlyList _advisories; + + public StubAdvisoryStore(params Advisory[] advisories) + { + _advisories = advisories; + } + + public void SetAdvisories(params Advisory[] advisories) + { + _advisories = advisories; + } + + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + => Task.FromResult(_advisories); + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + => Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + => Task.CompletedTask; + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) + { + return EnumerateAsync(cancellationToken); + + async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) + { + foreach (var advisory in _advisories) + { + ct.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + } + } + + private sealed class InMemoryExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; + + public Task FindAsync(string id, CancellationToken cancellationToken) + => Task.FromResult(_record); + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now; + + public TestTimeProvider(DateTimeOffset start) => _now = start; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan delta) => _now = _now.Add(delta); + } + + private sealed class StubTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + + public StubTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + var payload = new byte[] { 0x1, 0x2, 0x3, 0x4 }; + File.WriteAllBytes(archivePath, payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(payload)).ToLowerInvariant(); + var length = payload.Length; + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + length, + _metadata, + workingDirectory)); + } + } + + private sealed class RecordingTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + private readonly List _manifestDigests = new(); + + public RecordingTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + } + + public IReadOnlyList ManifestDigests => _manifestDigests; + public string[]? LastRelativePaths { get; private set; } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + LastRelativePaths = jsonTree.Files.Select(static file => file.RelativePath).ToArray(); + + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + var payload = new byte[] { 0x5, 0x6, 0x7, 0x8 }; + File.WriteAllBytes(archivePath, payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(payload)).ToLowerInvariant(); + _manifestDigests.Add(digest); + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + payload.Length, + _metadata, + workingDirectory)); + } + } + + private sealed record RunArtifacts( + string ExportId, + string ManifestDigest, + string IndexJson, + string MetadataJson, + string ManifestJson, + IReadOnlyDictionary Blobs); + + private async Task RunDeterministicExportAsync(IReadOnlyList advisories) + { + var workspace = Path.Combine(_root, $"deterministic-{Guid.NewGuid():N}"); + var jsonRoot = Path.Combine(workspace, "tree"); + Directory.CreateDirectory(workspace); + + var advisoryStore = new StubAdvisoryStore(advisories.ToArray()); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = workspace, + ReferencePrefix = "example/trivy", + KeepWorkingTree = true, + Json = new JsonExportOptions + { + OutputRoot = jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var exportedAt = DateTimeOffset.Parse("2024-10-01T00:00:00Z", CultureInfo.InvariantCulture); + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(exportedAt); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-10-02T00:00:00Z", + UpdatedAt = "2024-10-01T00:00:00Z", + }); + + var builder = new DeterministicTrivyDbBuilder(workspace, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = exportedAt.ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + var layoutPath = Path.Combine(workspace, exportId); + + var indexJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "index.json"), Encoding.UTF8); + var metadataJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "metadata.json"), Encoding.UTF8); + + using var indexDoc = JsonDocument.Parse(indexJson); + var manifestNode = indexDoc.RootElement.GetProperty("manifests")[0]; + var manifestDigest = manifestNode.GetProperty("digest").GetString()!; + + var manifestHex = manifestDigest[7..]; + var manifestJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "blobs", "sha256", manifestHex), Encoding.UTF8); + + var blobs = new Dictionary(StringComparer.Ordinal); + var blobsRoot = Path.Combine(layoutPath, "blobs", "sha256"); + foreach (var file in Directory.GetFiles(blobsRoot)) + { + var name = Path.GetFileName(file); + var content = await File.ReadAllBytesAsync(file); + blobs[name] = content; + } + + Directory.Delete(workspace, recursive: true); + + return new RunArtifacts(exportId, manifestDigest, indexJson, metadataJson, manifestJson, blobs); + } + + private sealed class DeterministicTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + private readonly byte[] _payload; + + public DeterministicTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + _payload = new byte[] { 0x21, 0x22, 0x23, 0x24, 0x25 }; + } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + File.WriteAllBytes(archivePath, _payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(_payload)).ToLowerInvariant(); + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + _payload.Length, + _metadata, + workingDirectory)); + } + } + + private sealed class StubTrivyDbOrasPusher : ITrivyDbOrasPusher + { + public List<(string Layout, string Reference, string ExportId)> Pushes { get; } = new(); + + public Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken) + { + Pushes.Add((layoutPath, reference, exportId)); + return Task.CompletedTask; + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs new file mode 100644 index 00000000..ce615979 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Security.Cryptography; +using StellaOps.Feedser.Exporter.TrivyDb; + +namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbOciWriterTests : IDisposable +{ + private readonly string _root; + + public TrivyDbOciWriterTests() + { + _root = Directory.CreateTempSubdirectory("feedser-trivy-oci-tests").FullName; + } + + [Fact] + public async Task WritesOciLayoutWithManifestIndex() + { + var metadata = Encoding.UTF8.GetBytes("{\"generatedAt\":\"2024-08-01T00:00:00Z\",\"schema\":1}"); + var archive = Enumerable.Range(0, 128).Select(static b => (byte)b).ToArray(); + var generatedAt = DateTimeOffset.Parse("2024-08-01T00:00:00Z"); + var archivePath = Path.Combine(_root, "db.bin"); + File.WriteAllBytes(archivePath, archive); + var archiveDigest = ComputeDigest(archive); + var request = new TrivyDbPackageRequest(metadata, archivePath, archiveDigest, archive.LongLength, generatedAt, "2024.08.01"); + + var builder = new TrivyDbPackageBuilder(); + var package = builder.BuildPackage(request); + + var writer = new TrivyDbOciWriter(); + var result = await writer.WriteAsync(package, Path.Combine(_root, "oci"), "feedser:v2024.08.01", CancellationToken.None); + + Assert.Equal(package.Manifest.Layers[0].Digest, package.Config.DatabaseDigest); + Assert.NotEmpty(result.BlobDigests); + Assert.Contains(result.ManifestDigest, result.BlobDigests); + + var layoutPath = Path.Combine(result.RootDirectory, "oci-layout"); + Assert.True(File.Exists(layoutPath)); + var layoutJson = await File.ReadAllTextAsync(layoutPath, CancellationToken.None); + Assert.Contains("\"imageLayoutVersion\":\"1.0.0\"", layoutJson, StringComparison.Ordinal); + + var metadataPath = Path.Combine(result.RootDirectory, "metadata.json"); + Assert.True(File.Exists(metadataPath)); + var roundTripMetadata = await File.ReadAllBytesAsync(metadataPath, CancellationToken.None); + Assert.Equal(metadata, roundTripMetadata); + + var indexPath = Path.Combine(result.RootDirectory, "index.json"); + Assert.True(File.Exists(indexPath)); + using var indexDocument = JsonDocument.Parse(await File.ReadAllBytesAsync(indexPath, CancellationToken.None)); + var manifestElement = indexDocument.RootElement.GetProperty("manifests")[0]; + Assert.Equal(result.ManifestDigest, manifestElement.GetProperty("digest").GetString()); + Assert.Equal(TrivyDbMediaTypes.OciManifest, manifestElement.GetProperty("mediaType").GetString()); + Assert.Equal("feedser:v2024.08.01", manifestElement.GetProperty("annotations").GetProperty("org.opencontainers.image.ref.name").GetString()); + + var manifestPath = Path.Combine(result.RootDirectory, "blobs", "sha256", result.ManifestDigest.Split(':')[1]); + var manifestBytes = await File.ReadAllBytesAsync(manifestPath, CancellationToken.None); + using var manifestDocument = JsonDocument.Parse(manifestBytes); + var configDescriptor = manifestDocument.RootElement.GetProperty("config"); + Assert.Equal(package.Manifest.Config.Digest, configDescriptor.GetProperty("digest").GetString()); + Assert.Equal(package.Manifest.Config.MediaType, configDescriptor.GetProperty("mediaType").GetString()); + var layer = manifestDocument.RootElement.GetProperty("layers")[0]; + Assert.Equal(package.Manifest.Layers[0].Digest, layer.GetProperty("digest").GetString()); + Assert.Equal(package.Manifest.Layers[0].MediaType, layer.GetProperty("mediaType").GetString()); + + foreach (var digest in package.Blobs.Keys) + { + var blobPath = Path.Combine(result.RootDirectory, "blobs", "sha256", digest.Split(':')[1]); + Assert.True(File.Exists(blobPath)); + } + } + + [Fact] + public async Task ThrowsOnUnsupportedDigest() + { + var package = new TrivyDbPackage( + new OciManifest(2, TrivyDbMediaTypes.OciManifest, new OciDescriptor(TrivyDbMediaTypes.TrivyConfig, "sha256:abcd", 4), Array.Empty()), + new TrivyConfigDocument(TrivyDbMediaTypes.TrivyConfig, DateTimeOffset.UtcNow, "1", "sha256:abcd", 4), + new Dictionary + { + ["md5:deadbeef"] = TrivyDbBlob.FromBytes(new byte[] { 1, 2, 3, 4 }), + }, + new byte[] { 123 }); + + var writer = new TrivyDbOciWriter(); + await Assert.ThrowsAsync(() => writer.WriteAsync(package, Path.Combine(_root, "invalid"), "feedser:bad", CancellationToken.None)); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // ignore cleanup issues + } + } + + private static string ComputeDigest(byte[] payload) + { + var hash = SHA256.HashData(payload); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs new file mode 100644 index 00000000..dcd4a6ab --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs @@ -0,0 +1,93 @@ +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using StellaOps.Feedser.Exporter.TrivyDb; + +namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbPackageBuilderTests +{ + [Fact] + public void BuildsOciManifestWithExpectedMediaTypes() + { + var metadata = Encoding.UTF8.GetBytes("{\"generatedAt\":\"2024-07-15T12:00:00Z\"}"); + var archive = Enumerable.Range(0, 256).Select(static b => (byte)b).ToArray(); + var archivePath = Path.GetTempFileName(); + File.WriteAllBytes(archivePath, archive); + var archiveDigest = ComputeDigest(archive); + + try + { + var request = new TrivyDbPackageRequest( + metadata, + archivePath, + archiveDigest, + archive.LongLength, + DateTimeOffset.Parse("2024-07-15T12:00:00Z"), + "2024.07.15"); + + var builder = new TrivyDbPackageBuilder(); + var package = builder.BuildPackage(request); + + Assert.Equal(TrivyDbMediaTypes.OciManifest, package.Manifest.MediaType); + Assert.Equal(TrivyDbMediaTypes.TrivyConfig, package.Manifest.Config.MediaType); + var layer = Assert.Single(package.Manifest.Layers); + Assert.Equal(TrivyDbMediaTypes.TrivyLayer, layer.MediaType); + + var configBytes = JsonSerializer.SerializeToUtf8Bytes(package.Config, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + var expectedConfigDigest = ComputeDigest(configBytes); + Assert.Equal(expectedConfigDigest, package.Manifest.Config.Digest); + + Assert.Equal(archiveDigest, layer.Digest); + Assert.True(package.Blobs.ContainsKey(archiveDigest)); + Assert.Equal(archive.LongLength, package.Blobs[archiveDigest].Length); + Assert.True(package.Blobs.ContainsKey(expectedConfigDigest)); + Assert.Equal(metadata, package.MetadataJson.ToArray()); + } + finally + { + if (File.Exists(archivePath)) + { + File.Delete(archivePath); + } + } + } + + [Fact] + public void ThrowsWhenMetadataMissing() + { + var builder = new TrivyDbPackageBuilder(); + var archivePath = Path.GetTempFileName(); + var archiveBytes = new byte[] { 1, 2, 3 }; + File.WriteAllBytes(archivePath, archiveBytes); + var digest = ComputeDigest(archiveBytes); + + try + { + Assert.Throws(() => builder.BuildPackage(new TrivyDbPackageRequest( + ReadOnlyMemory.Empty, + archivePath, + digest, + archiveBytes.LongLength, + DateTimeOffset.UtcNow, + "1"))); + } + finally + { + if (File.Exists(archivePath)) + { + File.Delete(archivePath); + } + } + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + return "sha256:" + hex.ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md b/src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md new file mode 100644 index 00000000..2897733c --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Exporter producing a Trivy-compatible database artifact for self-hosting or offline use. v0: JSON list + metadata; v1: integrate official trivy-db builder or write BoltDB directly; pack and optionally push via ORAS. +## Scope +- Read canonical advisories; serialize payload for builder or intermediate; write metadata.json (generatedAt, counts). +- Output root: exports/trivy/; deterministic path components. +- OCI/Trivy expectations: layer media type application/vnd.aquasec.trivy.db.layer.v1.tar+gzip; config media type application/vnd.aquasec.trivy.config.v1+json; tag (e.g., 2). +- Optional ORAS push; optional offline bundle (db.tar.gz + metadata.json). +- DI: TrivyExporter + Jobs.TrivyExportJob registered by TrivyExporterDependencyInjectionRoutine. +- Export_state recording: capture digests, counts, start/end timestamps for idempotent reruns and incremental packaging. +## Participants +- Storage.Mongo.AdvisoryStore as input. +- Core scheduler runs export job; WebService/Plugins trigger it. +- JSON exporter (optional precursor) if choosing the builder path. +## Interfaces & contracts +- IFeedExporter.Name = "trivy-db"; ExportAsync(IServiceProvider, CancellationToken). +- FeedserOptions.packaging.trivy governs repo/tag/publish/offline_bundle. +- Deterministic sorting and timestamp discipline (UTC; consider build reproducibility knobs). +## In/Out of scope +In: assembling builder inputs, packing tar.gz, pushing to registry when configured. +Out: signing (external pipeline), scanner behavior. +## Observability & security expectations +- Metrics: export.trivy.records, size_bytes, duration, oras.push.success/fail. +- Logs: export path, repo/tag, digest; redact credentials; backoff on push errors. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Exporter.TrivyDb.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs new file mode 100644 index 00000000..0f9854ba --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs @@ -0,0 +1,15 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Exporter.Json; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public interface ITrivyDbBuilder +{ + Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs new file mode 100644 index 00000000..d8a44075 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public interface ITrivyDbOrasPusher +{ + Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs new file mode 100644 index 00000000..58aeaec8 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record OciDescriptor( + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("digest")] string Digest, + [property: JsonPropertyName("size")] long Size, + [property: JsonPropertyName("annotations")] IReadOnlyDictionary? Annotations = null); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs new file mode 100644 index 00000000..eb00c179 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs @@ -0,0 +1,8 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record OciIndex( + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("manifests")] IReadOnlyList Manifests); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs new file mode 100644 index 00000000..ee99d638 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record OciManifest( + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("config")] OciDescriptor Config, + [property: JsonPropertyName("layers")] IReadOnlyList Layers); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj b/src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj new file mode 100644 index 00000000..fe28b621 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md b/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md new file mode 100644 index 00000000..4f7a7c4e --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Fix method name typo GetExportRoot' -> GetExportRoot|BE-Export|Exporters|DONE – `TrivyDbExportOptions.GetExportRoot` helper added with unit coverage.| +|Implement BoltDB builder integration (v0 via trivy-db CLI)|BE-Export|Env|DONE – `TrivyDbBoltBuilder` shells `trivy-db build` against our JSON tree with deterministic packaging.| +|Pack db.tar.gz + metadata.json|BE-Export|Exporters|DONE – Builder output re-packed with fixed timestamps and zeroed gzip mtime.| +|ORAS push support|BE-Export|Exporters|DONE – Optional `TrivyDbOrasPusher` shells `oras cp --from-oci-layout` with configurable args/env.| +|Offline bundle toggle|BE-Export|Exporters|DONE – Deterministic OCI layout bundle emitted when enabled.| +|Deterministic ordering of advisories|BE-Export|Models|DONE – exporter now loads advisories, sorts by advisoryKey, and emits sorted JSON trees with deterministic OCI payloads.| +|End-to-end tests with small dataset|QA|Exporters|DONE – added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.| +|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DOING – `ExportStateManager` keeps stable base export metadata; delta reset remains pending.| +|Streamed package building to avoid large copies|BE-Export|Exporters|DONE – metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.| +|Plan incremental/delta exports|BE-Export|Exporters|TODO – design reuse of existing blobs/layers when inputs unchanged instead of rewriting full trees each run.| diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs new file mode 100644 index 00000000..10e947c8 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs @@ -0,0 +1,11 @@ +using System; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyConfigDocument( + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt, + [property: JsonPropertyName("databaseVersion")] string DatabaseVersion, + [property: JsonPropertyName("databaseDigest")] string DatabaseDigest, + [property: JsonPropertyName("databaseSize")] long DatabaseSize); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs new file mode 100644 index 00000000..2ff58e60 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs @@ -0,0 +1,78 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbBlob +{ + private readonly Func> _openReadAsync; + + private TrivyDbBlob(Func> openReadAsync, long length) + { + _openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync)); + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + Length = length; + } + + public long Length { get; } + + public ValueTask OpenReadAsync(CancellationToken cancellationToken) + => _openReadAsync(cancellationToken); + + public static TrivyDbBlob FromBytes(ReadOnlyMemory payload) + { + if (payload.IsEmpty) + { + return new TrivyDbBlob(static _ => ValueTask.FromResult(Stream.Null), 0); + } + + if (MemoryMarshal.TryGetArray(payload, out ArraySegment segment) && segment.Array is not null && segment.Offset == 0) + { + return FromArray(segment.Array); + } + + return FromArray(payload.ToArray()); + } + + public static TrivyDbBlob FromFile(string path, long length) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("File path must be provided.", nameof(path)); + } + + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + return new TrivyDbBlob( + cancellationToken => ValueTask.FromResult(new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan)), + length); + } + + public static TrivyDbBlob FromArray(byte[] buffer) + { + if (buffer is null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + return new TrivyDbBlob( + _ => ValueTask.FromResult(new MemoryStream(buffer, writable: false)), + buffer.LongLength); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs new file mode 100644 index 00000000..e7723005 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs @@ -0,0 +1,376 @@ +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Formats.Tar; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Exporter.Json; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder +{ + private readonly TrivyDbExportOptions _options; + private readonly ILogger _logger; + + public TrivyDbBoltBuilder(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(jsonTree); + ArgumentException.ThrowIfNullOrEmpty(exportId); + + var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId); + var outputDir = Path.Combine(builderRoot, "out"); + Directory.CreateDirectory(outputDir); + + try + { + await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false); + } + catch + { + TryDeleteDirectory(builderRoot); + throw; + } + + var metadataPath = Path.Combine(outputDir, "metadata.json"); + var dbPath = Path.Combine(outputDir, "trivy.db"); + + if (!File.Exists(metadataPath)) + { + TryDeleteDirectory(builderRoot); + throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'."); + } + + if (!File.Exists(dbPath)) + { + TryDeleteDirectory(builderRoot); + throw new InvalidOperationException($"trivy.db not found at '{dbPath}'."); + } + + var archivePath = Path.Combine(builderRoot, "db.tar.gz"); + await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false); + + var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false); + var length = new FileInfo(archivePath).Length; + var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false); + + return new TrivyDbBuilderResult( + archivePath, + digest, + length, + builderMetadata, + builderRoot); + } + + private string PrepareBuilderRoot(string exportDirectory, string exportId) + { + var root = Path.Combine(exportDirectory, $".builder-{exportId}"); + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + + Directory.CreateDirectory(root); + return root; + } + + private static void TryDeleteDirectory(string directory) + { + try + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + catch + { + // ignore cleanup failures + } + } + + private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken) + { + var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions(); + var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath) + ? "trivy-db" + : builderOptions.ExecutablePath; + + var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List(); + var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary(StringComparer.OrdinalIgnoreCase); + + var startInfo = new ProcessStartInfo + { + FileName = executable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + startInfo.ArgumentList.Add("build"); + startInfo.ArgumentList.Add("--cache-dir"); + startInfo.ArgumentList.Add(cacheDir); + startInfo.ArgumentList.Add("--output-dir"); + startInfo.ArgumentList.Add(outputDir); + + if (builderOptions.UpdateInterval != default) + { + startInfo.ArgumentList.Add("--update-interval"); + startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval)); + } + + if (targets.Count > 0) + { + foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t))) + { + startInfo.ArgumentList.Add("--only-update"); + startInfo.ArgumentList.Add(target); + } + } + + if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory)) + { + startInfo.WorkingDirectory = builderOptions.WorkingDirectory; + } + + if (!builderOptions.InheritEnvironment) + { + startInfo.Environment.Clear(); + } + + foreach (var kvp in environment) + { + startInfo.Environment[kvp.Key] = kvp.Value; + } + + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false }; + + var stdOut = new StringBuilder(); + var stdErr = new StringBuilder(); + + var stdoutCompletion = new TaskCompletionSource(); + var stderrCompletion = new TaskCompletionSource(); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is null) + { + stdoutCompletion.TrySetResult(null); + } + else + { + stdOut.AppendLine(e.Data); + } + }; + + process.ErrorDataReceived += (_, e) => + { + if (e.Data is null) + { + stderrCompletion.TrySetResult(null); + } + else + { + stdErr.AppendLine(e.Data); + } + }; + + _logger.LogInformation("Running {Executable} to build Trivy DB", executable); + + try + { + if (!process.Start()) + { + throw new InvalidOperationException($"Failed to start '{executable}'."); + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to start '{executable}'.", ex); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var registration = cancellationToken.Register(() => + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch + { + // Ignore kill failures. + } + }); + +#if NET8_0_OR_GREATER + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); +#else + await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); +#endif + + await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + _logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString()); + throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); + } + + if (stdOut.Length > 0) + { + _logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString()); + } + + if (stdErr.Length > 0) + { + _logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString()); + } + } + + private static async Task CreateArchiveAsync( + string archivePath, + DateTimeOffset exportedAt, + string metadataPath, + string dbPath, + CancellationToken cancellationToken) + { + await using var archiveStream = new FileStream( + archivePath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true); + await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false); + + var timestamp = exportedAt.UtcDateTime; + foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name) + { + ModificationTime = timestamp, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead, + }; + + await using var source = new FileStream( + file.Path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + entry.DataStream = source; + writer.WriteEntry(entry); + } + + await writer.DisposeAsync().ConfigureAwait(false); + await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false); + } + + private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath) + { + yield return ("metadata.json", metadataPath); + yield return ("trivy.db", dbPath); + } + + private static async Task ComputeDigestAsync(string archivePath, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + archivePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + archivePath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None, + bufferSize: 8, + options: FileOptions.Asynchronous); + + if (stream.Length < 10) + { + return; + } + + stream.Position = 4; + var zeros = new byte[4]; + await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static string ToGoDuration(TimeSpan span) + { + if (span <= TimeSpan.Zero) + { + return "0s"; + } + + span = span.Duration(); + var builder = new StringBuilder(); + + var totalHours = (int)span.TotalHours; + if (totalHours > 0) + { + builder.Append(totalHours); + builder.Append('h'); + } + + var minutes = span.Minutes; + if (minutes > 0) + { + builder.Append(minutes); + builder.Append('m'); + } + + var seconds = span.Seconds + span.Milliseconds / 1000.0; + if (seconds > 0 || builder.Length == 0) + { + if (span.Milliseconds == 0) + { + builder.Append(span.Seconds); + } + else + { + builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture)); + } + builder.Append('s'); + } + + return builder.ToString(); + } + +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs new file mode 100644 index 00000000..cba55dbd --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs @@ -0,0 +1,10 @@ +using System; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbBuilderResult( + string ArchivePath, + string ArchiveDigest, + long ArchiveLength, + ReadOnlyMemory BuilderMetadata, + string WorkingDirectory); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs new file mode 100644 index 00000000..251e79b8 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs @@ -0,0 +1,30 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExportJob : IJob +{ + public const string JobKind = "export:trivy-db"; + public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20); + public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10); + + private readonly TrivyDbFeedExporter _exporter; + private readonly ILogger _logger; + + public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger logger) + { + _exporter = exporter ?? throw new ArgumentNullException(nameof(exporter)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + _logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId); + await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs new file mode 100644 index 00000000..25dc5a8c --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public enum TrivyDbExportMode +{ + Full, + Delta, + Skip, +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs new file mode 100644 index 00000000..e9c4f5d4 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs @@ -0,0 +1,80 @@ +using System; +using System.IO; +using System.Collections.Generic; +using StellaOps.Feedser.Exporter.Json; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExportOptions +{ + public string OutputRoot { get; set; } = Path.Combine("exports", "trivy"); + + public string ReferencePrefix { get; set; } = "feedser/trivy"; + + public string TagFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; + + public string DatabaseVersionFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; + + public bool KeepWorkingTree { get; set; } + + public string? TargetRepository { get; set; } + + public JsonExportOptions Json { get; set; } = new() + { + OutputRoot = Path.Combine("exports", "trivy", "tree") + }; + + public TrivyDbBuilderOptions Builder { get; set; } = new(); + + public TrivyDbOrasOptions Oras { get; set; } = new(); + + public TrivyDbOfflineBundleOptions OfflineBundle { get; set; } = new(); + + public string GetExportRoot(string exportId) + { + ArgumentException.ThrowIfNullOrEmpty(exportId); + var root = Path.GetFullPath(OutputRoot); + return Path.Combine(root, exportId); + } +} + +public sealed class TrivyDbBuilderOptions +{ + public string ExecutablePath { get; set; } = "trivy-db"; + + public string? WorkingDirectory { get; set; } + + public TimeSpan UpdateInterval { get; set; } = TimeSpan.FromHours(24); + + public List OnlyUpdateTargets { get; set; } = new(); + + public Dictionary Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + public bool InheritEnvironment { get; set; } = true; +} + +public sealed class TrivyDbOrasOptions +{ + public bool Enabled { get; set; } + + public string ExecutablePath { get; set; } = "oras"; + + public string? WorkingDirectory { get; set; } + + public bool InheritEnvironment { get; set; } = true; + + public List AdditionalArguments { get; set; } = new(); + + public Dictionary Environment { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + public bool SkipTlsVerify { get; set; } + + public bool UseHttp { get; set; } +} + +public sealed class TrivyDbOfflineBundleOptions +{ + public bool Enabled { get; set; } + + public string? FileName { get; set; } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs new file mode 100644 index 00000000..d889eae1 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbExportPlan( + TrivyDbExportMode Mode, + string TreeDigest, + string? BaseExportId, + string? BaseManifestDigest); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs new file mode 100644 index 00000000..0f9fb0c2 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs @@ -0,0 +1,33 @@ +using System; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExportPlanner +{ + public TrivyDbExportPlan CreatePlan(ExportStateRecord? existingState, string treeDigest) + { + ArgumentException.ThrowIfNullOrEmpty(treeDigest); + + if (existingState is null) + { + return new TrivyDbExportPlan(TrivyDbExportMode.Full, treeDigest, BaseExportId: null, BaseManifestDigest: null); + } + + if (string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal)) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Skip, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest); + } + + // Placeholder for future delta support – current behavior always rebuilds when tree changes. + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs new file mode 100644 index 00000000..c55cb223 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs @@ -0,0 +1,64 @@ +using System; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:exporters:trivyDb"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Bind(configuration.GetSection(ConfigurationSection)) + .PostConfigure(static options => + { + options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy")); + options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree")); + options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat; + options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat; + options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "feedser/trivy" : options.ReferencePrefix; + }); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddTransient(); + + services.PostConfigure(options => + { + if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind)) + { + options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition( + TrivyDbExportJob.JobKind, + typeof(TrivyDbExportJob), + TrivyDbExportJob.DefaultTimeout, + TrivyDbExportJob.DefaultLeaseDuration, + null, + true); + } + }); + + return services; + } + + private static string Normalize(string? value, string fallback) + => string.IsNullOrWhiteSpace(value) ? fallback : value; +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs new file mode 100644 index 00000000..aab67973 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs @@ -0,0 +1,23 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbExporterPlugin : IExporterPlugin +{ + public string Name => TrivyDbFeedExporter.ExporterName; + + public bool IsAvailable(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetService() is not null; + } + + public IFeedExporter Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs new file mode 100644 index 00000000..773b72c8 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs @@ -0,0 +1,384 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using System.Formats.Tar; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Exporter.Json; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbFeedExporter : IFeedExporter +{ + public const string ExporterName = "trivy-db"; + public const string ExporterId = "export:trivy-db"; + + private readonly IAdvisoryStore _advisoryStore; + private readonly IJsonExportPathResolver _pathResolver; + private readonly TrivyDbExportOptions _options; + private readonly TrivyDbPackageBuilder _packageBuilder; + private readonly TrivyDbOciWriter _ociWriter; + private readonly ExportStateManager _stateManager; + private readonly TrivyDbExportPlanner _exportPlanner; + private readonly ITrivyDbBuilder _builder; + private readonly ITrivyDbOrasPusher _orasPusher; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _exporterVersion; + + public TrivyDbFeedExporter( + IAdvisoryStore advisoryStore, + IJsonExportPathResolver pathResolver, + IOptions options, + TrivyDbPackageBuilder packageBuilder, + TrivyDbOciWriter ociWriter, + ExportStateManager stateManager, + TrivyDbExportPlanner exportPlanner, + ITrivyDbBuilder builder, + ITrivyDbOrasPusher orasPusher, + ILogger logger, + TimeProvider? timeProvider = null) + { + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _packageBuilder = packageBuilder ?? throw new ArgumentNullException(nameof(packageBuilder)); + _ociWriter = ociWriter ?? throw new ArgumentNullException(nameof(ociWriter)); + _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); + _exportPlanner = exportPlanner ?? throw new ArgumentNullException(nameof(exportPlanner)); + _builder = builder ?? throw new ArgumentNullException(nameof(builder)); + _orasPusher = orasPusher ?? throw new ArgumentNullException(nameof(orasPusher)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _exporterVersion = ExporterVersion.GetVersion(typeof(TrivyDbFeedExporter)); + } + + public string Name => ExporterName; + + public async Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var exportedAt = _timeProvider.GetUtcNow(); + var exportId = exportedAt.ToString(_options.TagFormat, CultureInfo.InvariantCulture); + var reference = $"{_options.ReferencePrefix}:{exportId}"; + + _logger.LogInformation("Starting Trivy DB export {ExportId}", exportId); + + var jsonBuilder = new JsonExportSnapshotBuilder(_options.Json, _pathResolver); + var advisories = await LoadAdvisoriesAsync(cancellationToken).ConfigureAwait(false); + var jsonResult = await jsonBuilder.WriteAsync(advisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Prepared Trivy JSON tree {ExportId} with {AdvisoryCount} advisories ({Bytes} bytes)", + exportId, + jsonResult.AdvisoryCount, + jsonResult.TotalBytes); + + var treeDigest = ExportDigestCalculator.ComputeTreeDigest(jsonResult); + var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); + var plan = _exportPlanner.CreatePlan(existingState, treeDigest); + + if (plan.Mode == TrivyDbExportMode.Skip) + { + _logger.LogInformation( + "Trivy DB export {ExportId} unchanged from base {BaseExport}; skipping OCI packaging.", + exportId, + plan.BaseExportId ?? "(none)"); + + if (!_options.KeepWorkingTree) + { + TryDeleteDirectory(jsonResult.ExportDirectory); + } + + return; + } + + var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false); + var metadataBytes = CreateMetadataJson(builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt); + + try + { + var package = _packageBuilder.BuildPackage(new TrivyDbPackageRequest( + metadataBytes, + builderResult.ArchivePath, + builderResult.ArchiveDigest, + builderResult.ArchiveLength, + exportedAt, + exportedAt.ToString(_options.DatabaseVersionFormat, CultureInfo.InvariantCulture))); + + var destination = _options.GetExportRoot(exportId); + var ociResult = await _ociWriter.WriteAsync(package, destination, reference, cancellationToken).ConfigureAwait(false); + + if (_options.Oras.Enabled) + { + await _orasPusher.PushAsync(destination, reference, exportId, cancellationToken).ConfigureAwait(false); + } + + _logger.LogInformation( + "Trivy DB export {ExportId} wrote manifest {ManifestDigest}", + exportId, + ociResult.ManifestDigest); + + await _stateManager.StoreFullExportAsync( + ExporterId, + exportId, + ociResult.ManifestDigest, + cursor: treeDigest, + targetRepository: _options.TargetRepository, + exporterVersion: _exporterVersion, + cancellationToken: cancellationToken).ConfigureAwait(false); + + await CreateOfflineBundleAsync(destination, exportId, exportedAt, cancellationToken).ConfigureAwait(false); + } + finally + { + TryDeleteDirectory(builderResult.WorkingDirectory); + } + + if (!_options.KeepWorkingTree) + { + TryDeleteDirectory(jsonResult.ExportDirectory); + } + } + + private async Task> LoadAdvisoriesAsync(CancellationToken cancellationToken) + { + var advisories = new List(); + await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken).ConfigureAwait(false)) + { + if (advisory is null) + { + continue; + } + + advisories.Add(advisory); + } + + advisories.Sort(static (left, right) => string.CompareOrdinal(left.AdvisoryKey, right.AdvisoryKey)); + return advisories; + } + + private byte[] CreateMetadataJson( + ReadOnlyMemory builderMetadata, + string treeDigest, + JsonExportResult result, + DateTimeOffset exportedAt) + { + var metadata = new TrivyMetadata + { + GeneratedAt = exportedAt.UtcDateTime, + AdvisoryCount = result.AdvisoryCount, + TreeDigest = treeDigest, + TreeBytes = result.TotalBytes, + ExporterVersion = _exporterVersion, + Builder = ParseBuilderMetadata(builderMetadata.Span), + }; + + return JsonSerializer.SerializeToUtf8Bytes(metadata, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }); + } + + private static BuilderMetadata? ParseBuilderMetadata(ReadOnlySpan payload) + { + if (payload.IsEmpty) + { + return null; + } + + try + { + return JsonSerializer.Deserialize(payload, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + }); + } + catch + { + return null; + } + } + + private async Task CreateOfflineBundleAsync(string layoutPath, string exportId, DateTimeOffset exportedAt, CancellationToken cancellationToken) + { + if (!_options.OfflineBundle.Enabled) + { + return; + } + + var parent = Path.GetDirectoryName(layoutPath) ?? layoutPath; + var fileName = string.IsNullOrWhiteSpace(_options.OfflineBundle.FileName) + ? $"{exportId}.offline.tar.gz" + : _options.OfflineBundle.FileName.Replace("{exportId}", exportId, StringComparison.Ordinal); + + var bundlePath = Path.IsPathRooted(fileName) ? fileName : Path.Combine(parent, fileName); + Directory.CreateDirectory(Path.GetDirectoryName(bundlePath)!); + + if (File.Exists(bundlePath)) + { + File.Delete(bundlePath); + } + + var normalizedRoot = Path.GetFullPath(layoutPath); + var directories = Directory.GetDirectories(normalizedRoot, "*", SearchOption.AllDirectories) + .Select(dir => NormalizeTarPath(normalizedRoot, dir) + "/") + .OrderBy(static path => path, StringComparer.Ordinal) + .ToArray(); + + var files = Directory.GetFiles(normalizedRoot, "*", SearchOption.AllDirectories) + .Select(file => NormalizeTarPath(normalizedRoot, file)) + .OrderBy(static path => path, StringComparer.Ordinal) + .ToArray(); + + await using (var archiveStream = new FileStream( + bundlePath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan)) + await using (var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true)) + await using (var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false)) + { + var timestamp = exportedAt.UtcDateTime; + + foreach (var directory in directories) + { + var entry = new PaxTarEntry(TarEntryType.Directory, directory) + { + ModificationTime = timestamp, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }; + + writer.WriteEntry(entry); + } + + foreach (var relativePath in files) + { + var fullPath = Path.Combine(normalizedRoot, relativePath.Replace('/', Path.DirectorySeparatorChar)); + var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath) + { + ModificationTime = timestamp, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | + UnixFileMode.GroupRead | + UnixFileMode.OtherRead, + }; + + await using var source = new FileStream( + fullPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + entry.DataStream = source; + writer.WriteEntry(entry); + } + } + + await ZeroGzipMtimeAsync(bundlePath, cancellationToken).ConfigureAwait(false); + + var digest = await ComputeSha256Async(bundlePath, cancellationToken).ConfigureAwait(false); + var length = new FileInfo(bundlePath).Length; + _logger.LogInformation("Wrote offline bundle {BundlePath} ({Length} bytes, digest {Digest})", bundlePath, length, digest); + } + + private static void TryDeleteDirectory(string directory) + { + try + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + catch + { + // Best effort cleanup – ignore failures. + } + } + + private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + archivePath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None, + bufferSize: 8, + options: FileOptions.Asynchronous); + + if (stream.Length < 10) + { + return; + } + + stream.Position = 4; + var zeros = new byte[4]; + await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ComputeSha256Async(string path, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static string NormalizeTarPath(string root, string fullPath) + { + var relative = Path.GetRelativePath(root, fullPath); + var normalized = relative.Replace(Path.DirectorySeparatorChar, '/'); + return string.IsNullOrEmpty(normalized) ? "." : normalized; + } + + private sealed class TrivyMetadata + { + public DateTime GeneratedAt { get; set; } + + public int AdvisoryCount { get; set; } + + public string TreeDigest { get; set; } = string.Empty; + + public long TreeBytes { get; set; } + + public string ExporterVersion { get; set; } = string.Empty; + + public BuilderMetadata? Builder { get; set; } + } + + private sealed class BuilderMetadata + { + [JsonPropertyName("Version")] + public int Version { get; set; } + + public DateTime NextUpdate { get; set; } + + public DateTime UpdatedAt { get; set; } + + public DateTime? DownloadedAt { get; set; } + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs new file mode 100644 index 00000000..cf667440 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public static class TrivyDbMediaTypes +{ + public const string OciManifest = "application/vnd.oci.image.manifest.v1+json"; + public const string OciImageIndex = "application/vnd.oci.image.index.v1+json"; + public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json"; + public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip"; +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs new file mode 100644 index 00000000..3a22a7ab --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs @@ -0,0 +1,8 @@ +using System.Collections.Generic; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbOciWriteResult( + string RootDirectory, + string ManifestDigest, + IReadOnlyCollection BlobDigests); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs new file mode 100644 index 00000000..c599a328 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +/// +/// Writes a Trivy DB package to an OCI image layout directory with deterministic content. +/// +public sealed class TrivyDbOciWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + + private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}"); + + public async Task WriteAsync( + TrivyDbPackage package, + string destination, + string reference, + CancellationToken cancellationToken) + { + if (package is null) + { + throw new ArgumentNullException(nameof(package)); + } + + if (string.IsNullOrWhiteSpace(destination)) + { + throw new ArgumentException("Destination directory must be provided.", nameof(destination)); + } + + if (string.IsNullOrWhiteSpace(reference)) + { + throw new ArgumentException("Reference tag must be provided.", nameof(reference)); + } + + var root = Path.GetFullPath(destination); + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + + Directory.CreateDirectory(root); + var timestamp = package.Config.GeneratedAt.UtcDateTime; + + await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false); + await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false); + + var blobsRoot = Path.Combine(root, "blobs", "sha256"); + Directory.CreateDirectory(blobsRoot); + Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp); + Directory.SetLastWriteTimeUtc(blobsRoot, timestamp); + + var writtenDigests = new HashSet(StringComparer.Ordinal); + foreach (var pair in package.Blobs) + { + if (writtenDigests.Add(pair.Key)) + { + await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false); + } + } + + var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions); + var manifestDigest = ComputeDigest(manifestBytes); + if (writtenDigests.Add(manifestDigest)) + { + await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false); + } + + var manifestDescriptor = new OciDescriptor( + TrivyDbMediaTypes.OciManifest, + manifestDigest, + manifestBytes.LongLength, + new Dictionary + { + ["org.opencontainers.image.ref.name"] = reference, + }); + var index = new OciIndex(2, new[] { manifestDescriptor }); + var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions); + await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false); + + Directory.SetLastWriteTimeUtc(root, timestamp); + + var blobDigests = writtenDigests.ToArray(); + Array.Sort(blobDigests, StringComparer.Ordinal); + return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests); + } + + private static async Task WriteFileAsync(string path, ReadOnlyMemory bytes, DateTime utcTimestamp, CancellationToken cancellationToken) + { + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + Directory.SetLastWriteTimeUtc(directory, utcTimestamp); + } + + await using var destination = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(path, utcTimestamp); + } + + private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken) + { + var fileName = ResolveDigestFileName(digest); + var path = Path.Combine(blobsRoot, fileName); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + Directory.SetLastWriteTimeUtc(directory, utcTimestamp); + } + + await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false); + await using var destination = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(path, utcTimestamp); + } + + private static string ResolveDigestFileName(string digest) + { + if (!digest.StartsWith("sha256:", StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'."); + } + + var hex = digest[7..]; + if (hex.Length == 0) + { + throw new InvalidOperationException("Digest hex component cannot be empty."); + } + + return hex; + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = System.Security.Cryptography.SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + Span buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex + buffer[0] = 's'; + buffer[1] = 'h'; + buffer[2] = 'a'; + buffer[3] = '2'; + buffer[4] = '5'; + buffer[5] = '6'; + buffer[6] = ':'; + for (var i = 0; i < hex.Length; i++) + { + buffer[7 + i] = char.ToLowerInvariant(hex[i]); + } + + return new string(buffer); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs new file mode 100644 index 00000000..5a723a01 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs @@ -0,0 +1,209 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher +{ + private readonly TrivyDbExportOptions _options; + private readonly ILogger _logger; + + public TrivyDbOrasPusher(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken) + { + var orasOptions = _options.Oras; + if (!orasOptions.Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(reference)) + { + throw new InvalidOperationException("ORAS push requested but reference is empty."); + } + + if (!Directory.Exists(layoutPath)) + { + throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist."); + } + + var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath; + var tag = ResolveTag(reference, exportId); + var layoutReference = $"{layoutPath}:{tag}"; + + var startInfo = new ProcessStartInfo + { + FileName = executable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + startInfo.ArgumentList.Add("cp"); + startInfo.ArgumentList.Add("--from-oci-layout"); + startInfo.ArgumentList.Add(layoutReference); + if (orasOptions.SkipTlsVerify) + { + startInfo.ArgumentList.Add("--insecure"); + } + if (orasOptions.UseHttp) + { + startInfo.ArgumentList.Add("--plain-http"); + } + + if (orasOptions.AdditionalArguments is { Count: > 0 }) + { + foreach (var arg in orasOptions.AdditionalArguments) + { + if (!string.IsNullOrWhiteSpace(arg)) + { + startInfo.ArgumentList.Add(arg); + } + } + } + + startInfo.ArgumentList.Add(reference); + + if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory)) + { + startInfo.WorkingDirectory = orasOptions.WorkingDirectory; + } + + if (!orasOptions.InheritEnvironment) + { + startInfo.Environment.Clear(); + } + + if (orasOptions.Environment is { Count: > 0 }) + { + foreach (var kvp in orasOptions.Environment) + { + if (!string.IsNullOrEmpty(kvp.Key)) + { + startInfo.Environment[kvp.Key] = kvp.Value; + } + } + } + + using var process = new Process { StartInfo = startInfo }; + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + var stdoutCompletion = new TaskCompletionSource(); + var stderrCompletion = new TaskCompletionSource(); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is null) + { + stdoutCompletion.TrySetResult(null); + } + else + { + stdout.AppendLine(e.Data); + } + }; + + process.ErrorDataReceived += (_, e) => + { + if (e.Data is null) + { + stderrCompletion.TrySetResult(null); + } + else + { + stderr.AppendLine(e.Data); + } + }; + + _logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable); + + try + { + if (!process.Start()) + { + throw new InvalidOperationException($"Failed to start '{executable}'."); + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to start '{executable}'.", ex); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var registration = cancellationToken.Register(() => + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch + { + // ignore + } + }); + +#if NET8_0_OR_GREATER + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); +#else + await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); +#endif + + await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + _logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString()); + throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); + } + + if (stdout.Length > 0) + { + _logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString()); + } + + if (stderr.Length > 0) + { + _logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString()); + } + } + + private static string ResolveTag(string reference, string fallback) + { + if (string.IsNullOrWhiteSpace(reference)) + { + return fallback; + } + + var atIndex = reference.IndexOf('@'); + if (atIndex >= 0) + { + reference = reference[..atIndex]; + } + + var slashIndex = reference.LastIndexOf('/'); + var colonIndex = reference.LastIndexOf(':'); + if (colonIndex > slashIndex && colonIndex >= 0) + { + return reference[(colonIndex + 1)..]; + } + + return fallback; + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs new file mode 100644 index 00000000..c2d842da --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbPackage( + OciManifest Manifest, + TrivyConfigDocument Config, + IReadOnlyDictionary Blobs, + ReadOnlyMemory MetadataJson); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs new file mode 100644 index 00000000..2f5c8a45 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed class TrivyDbPackageBuilder +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + + public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + if (request.MetadataJson.IsEmpty) + { + throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request)); + } + + if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath)) + { + throw new ArgumentException("Database archive path must be provided.", nameof(request)); + } + + if (!File.Exists(request.DatabaseArchivePath)) + { + throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath); + } + + if (string.IsNullOrWhiteSpace(request.DatabaseDigest)) + { + throw new ArgumentException("Database archive digest must be provided.", nameof(request)); + } + + if (request.DatabaseLength < 0) + { + throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength)); + } + + var metadataBytes = request.MetadataJson; + var generatedAt = request.GeneratedAt.ToUniversalTime(); + var configDocument = new TrivyConfigDocument( + TrivyDbMediaTypes.TrivyConfig, + generatedAt, + request.DatabaseVersion, + request.DatabaseDigest, + request.DatabaseLength); + + var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions); + var configDigest = ComputeDigest(configBytes); + + var configDescriptor = new OciDescriptor( + TrivyDbMediaTypes.TrivyConfig, + configDigest, + configBytes.LongLength, + new Dictionary + { + ["org.opencontainers.image.title"] = "config.json", + }); + + var layerDescriptor = new OciDescriptor( + TrivyDbMediaTypes.TrivyLayer, + request.DatabaseDigest, + request.DatabaseLength, + new Dictionary + { + ["org.opencontainers.image.title"] = "db.tar.gz", + }); + + var manifest = new OciManifest( + 2, + TrivyDbMediaTypes.OciManifest, + configDescriptor, + ImmutableArray.Create(layerDescriptor)); + + var blobs = new SortedDictionary(StringComparer.Ordinal) + { + [configDigest] = TrivyDbBlob.FromBytes(configBytes), + [request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength), + }; + + return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes); + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + Span buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex + buffer[0] = 's'; + buffer[1] = 'h'; + buffer[2] = 'a'; + buffer[3] = '2'; + buffer[4] = '5'; + buffer[5] = '6'; + buffer[6] = ':'; + for (var i = 0; i < hex.Length; i++) + { + buffer[7 + i] = char.ToLowerInvariant(hex[i]); + } + + return new string(buffer); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs new file mode 100644 index 00000000..e39618a2 --- /dev/null +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs @@ -0,0 +1,11 @@ +using System; + +namespace StellaOps.Feedser.Exporter.TrivyDb; + +public sealed record TrivyDbPackageRequest( + ReadOnlyMemory MetadataJson, + string DatabaseArchivePath, + string DatabaseDigest, + long DatabaseLength, + DateTimeOffset GeneratedAt, + string DatabaseVersion); diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs b/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs new file mode 100644 index 00000000..126676c9 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs @@ -0,0 +1,262 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Merge.Options; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class AdvisoryPrecedenceMergerTests +{ + [Fact] + public void Merge_PrefersVendorPrecedenceOverNvd() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + + var (redHat, nvd) = CreateVendorAndRegistryAdvisories(); + var expectedMergeTimestamp = timeProvider.GetUtcNow(); + + var merged = merger.Merge(new[] { nvd, redHat }); + + Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); + Assert.Equal("Red Hat Security Advisory", merged.Title); + Assert.Equal("Vendor-confirmed impact on RHEL 9.", merged.Summary); + Assert.Equal("high", merged.Severity); + Assert.Equal(redHat.Published, merged.Published); + Assert.Equal(redHat.Modified, merged.Modified); + Assert.Contains("RHSA-2025:0001", merged.Aliases); + Assert.Contains("CVE-2025-1000", merged.Aliases); + + var package = Assert.Single(merged.AffectedPackages); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", package.Identifier); + Assert.Empty(package.VersionRanges); // NVD range suppressed by vendor precedence + Assert.Contains(package.Statuses, status => status.Status == "known_affected"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); + + Assert.Contains(merged.CvssMetrics, metric => metric.Provenance.Source == "redhat"); + Assert.Contains(merged.CvssMetrics, metric => metric.Provenance.Source == "nvd"); + + var mergeProvenance = merged.Provenance.Single(p => p.Source == "merge"); + Assert.Equal("precedence", mergeProvenance.Kind); + Assert.Equal(expectedMergeTimestamp, mergeProvenance.RecordedAt); + Assert.Contains("redhat", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + Assert.Contains("nvd", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Merge_KevOnlyTogglesExploitKnown() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero)); + var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + + var nvdProvenance = new AdvisoryProvenance("nvd", "document", "https://nvd", timeProvider.GetUtcNow()); + var baseAdvisory = new Advisory( + "CVE-2025-2000", + "CVE-2025-2000", + "Base registry summary", + "en", + new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero), + new DateTimeOffset(2025, 1, 6, 0, 0, 0, TimeSpan.Zero), + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-2000" }, + references: Array.Empty(), + affectedPackages: new[] + { + new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:a:example:product:2.0:*:*:*:*:*:*:*", + null, + new[] + { + new AffectedVersionRange( + "semver", + "2.0.0", + "2.0.5", + null, + "<2.0.5", + new AdvisoryProvenance("nvd", "cpe_match", "product", timeProvider.GetUtcNow())) + }, + Array.Empty(), + new[] { nvdProvenance }) + }, + cvssMetrics: Array.Empty(), + provenance: new[] { nvdProvenance }); + + var kevProvenance = new AdvisoryProvenance("kev", "catalog", "CVE-2025-2000", timeProvider.GetUtcNow()); + var kevAdvisory = new Advisory( + "CVE-2025-2000", + "Known Exploited Vulnerability", + summary: null, + language: null, + published: null, + modified: null, + severity: null, + exploitKnown: true, + aliases: new[] { "KEV-CVE-2025-2000" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { kevProvenance }); + + var merged = merger.Merge(new[] { baseAdvisory, kevAdvisory }); + + Assert.True(merged.ExploitKnown); + Assert.Equal("medium", merged.Severity); // KEV must not override severity + Assert.Equal("Base registry summary", merged.Summary); + Assert.Contains("CVE-2025-2000", merged.Aliases); + Assert.Contains("KEV-CVE-2025-2000", merged.Aliases); + Assert.Contains(merged.Provenance, provenance => provenance.Source == "kev"); + Assert.Contains(merged.Provenance, provenance => provenance.Source == "merge"); + } + + [Fact] + public void Merge_RespectsConfiguredPrecedenceOverrides() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero)); + var options = new AdvisoryPrecedenceOptions + { + Ranks = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["nvd"] = 0, + ["redhat"] = 5, + } + }; + + var logger = new TestLogger(); + using var metrics = new MetricCollector("StellaOps.Feedser.Merge"); + + var merger = new AdvisoryPrecedenceMerger( + new AffectedPackagePrecedenceResolver(), + options, + timeProvider, + logger); + + var (redHat, nvd) = CreateVendorAndRegistryAdvisories(); + var merged = merger.Merge(new[] { redHat, nvd }); + + Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); + Assert.Equal("CVE-2025-1000", merged.Title); // NVD preferred + Assert.Equal("NVD summary", merged.Summary); + Assert.Equal("medium", merged.Severity); + + var package = Assert.Single(merged.AffectedPackages); + Assert.NotEmpty(package.VersionRanges); // Vendor range no longer overrides + Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); + + var overrideMeasurement = Assert.Single(metrics.Measurements, m => m.Name == "feedser.merge.overrides"); + Assert.Equal(1, overrideMeasurement.Value); + Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "primary_source" && string.Equals(tag.Value?.ToString(), "nvd", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "suppressed_source" && tag.Value?.ToString()?.Contains("redhat", StringComparison.OrdinalIgnoreCase) == true); + + var logEntry = Assert.Single(logger.Entries, entry => entry.EventId.Name == "AdvisoryOverride"); + Assert.Equal(LogLevel.Information, logEntry.Level); + Assert.NotNull(logEntry.StructuredState); + Assert.Contains(logEntry.StructuredState!, kvp => + (string.Equals(kvp.Key, "Override", StringComparison.Ordinal) || + string.Equals(kvp.Key, "@Override", StringComparison.Ordinal)) && + kvp.Value is not null); + } + + private static (Advisory Vendor, Advisory Registry) CreateVendorAndRegistryAdvisories() + { + var redHatPublished = new DateTimeOffset(2025, 1, 10, 0, 0, 0, TimeSpan.Zero); + var redHatModified = redHatPublished.AddDays(1); + var redHatProvenance = new AdvisoryProvenance("redhat", "advisory", "RHSA-2025:0001", redHatModified); + var redHatPackage = new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "rhel-9", + Array.Empty(), + new[] { new AffectedPackageStatus("known_affected", redHatProvenance) }, + new[] { redHatProvenance }); + var redHat = new Advisory( + "CVE-2025-1000", + "Red Hat Security Advisory", + "Vendor-confirmed impact on RHEL 9.", + "en", + redHatPublished, + redHatModified, + "high", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000", "RHSA-2025:0001" }, + references: new[] + { + new AdvisoryReference( + "https://access.redhat.com/errata/RHSA-2025:0001", + "advisory", + "redhat", + "Red Hat errata", + redHatProvenance) + }, + affectedPackages: new[] { redHatPackage }, + cvssMetrics: new[] + { + new CvssMetric( + "3.1", + "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + 9.8, + "critical", + new AdvisoryProvenance("redhat", "cvss", "RHSA-2025:0001", redHatModified)) + }, + provenance: new[] { redHatProvenance }); + + var nvdPublished = new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero); + var nvdModified = nvdPublished.AddDays(2); + var nvdProvenance = new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-1000", nvdModified); + var nvdPackage = new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "rhel-9", + new[] + { + new AffectedVersionRange( + "cpe", + null, + null, + null, + "<=9.0", + new AdvisoryProvenance("nvd", "cpe_match", "RHEL", nvdModified)) + }, + Array.Empty(), + new[] { nvdProvenance }); + var nvd = new Advisory( + "CVE-2025-1000", + "CVE-2025-1000", + "NVD summary", + "en", + nvdPublished, + nvdModified, + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000" }, + references: new[] + { + new AdvisoryReference( + "https://nvd.nist.gov/vuln/detail/CVE-2025-1000", + "advisory", + "nvd", + "NVD advisory", + nvdProvenance) + }, + affectedPackages: new[] { nvdPackage }, + cvssMetrics: new[] + { + new CvssMetric( + "3.1", + "CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:N", + 6.8, + "medium", + new AdvisoryProvenance("nvd", "cvss", "CVE-2025-1000", nvdModified)) + }, + provenance: new[] { nvdProvenance }); + + return (redHat, nvd); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs new file mode 100644 index 00000000..db2467c9 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs @@ -0,0 +1,88 @@ +using System; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class AffectedPackagePrecedenceResolverTests +{ + [Fact] + public void Merge_PrefersRedHatOverNvdForSameCpe() + { + var redHat = new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + platform: "RHEL 9", + versionRanges: Array.Empty(), + statuses: new[] + { + new AffectedPackageStatus( + status: "known_affected", + provenance: new AdvisoryProvenance("redhat", "oval", "RHEL-9", DateTimeOffset.Parse("2025-10-01T00:00:00Z"))) + }, + provenance: new[] + { + new AdvisoryProvenance("redhat", "oval", "RHEL-9", DateTimeOffset.Parse("2025-10-01T00:00:00Z")) + }); + + var nvd = new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + platform: "RHEL 9", + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "cpe", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: "<=9.0", + provenance: new AdvisoryProvenance("nvd", "cpe_match", "RHEL-9", DateTimeOffset.Parse("2025-09-30T00:00:00Z"))) + }, + provenance: new[] + { + new AdvisoryProvenance("nvd", "cpe_match", "RHEL-9", DateTimeOffset.Parse("2025-09-30T00:00:00Z")) + }); + + var resolver = new AffectedPackagePrecedenceResolver(); + var merged = resolver.Merge(new[] { nvd, redHat }); + + var package = Assert.Single(merged); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", package.Identifier); + Assert.Empty(package.VersionRanges); // NVD range overridden + Assert.Contains(package.Statuses, status => status.Status == "known_affected"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); + Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); + } + + [Fact] + public void Merge_KeepsNvdWhenNoHigherPrecedence() + { + var nvd = new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: "cpe:2.3:a:example:product:1.0:*:*:*:*:*:*:*", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: null, + fixedVersion: "1.0.1", + lastAffectedVersion: null, + rangeExpression: "<1.0.1", + provenance: new AdvisoryProvenance("nvd", "cpe_match", "product", DateTimeOffset.Parse("2025-09-01T00:00:00Z"))) + }, + provenance: new[] + { + new AdvisoryProvenance("nvd", "cpe_match", "product", DateTimeOffset.Parse("2025-09-01T00:00:00Z")) + }); + + var resolver = new AffectedPackagePrecedenceResolver(); + var merged = resolver.Merge(new[] { nvd }); + + var package = Assert.Single(merged); + Assert.Equal(nvd.Identifier, package.Identifier); + Assert.Equal(nvd.VersionRanges.Single().RangeExpression, package.VersionRanges.Single().RangeExpression); + Assert.Equal("nvd", package.Provenance.Single().Source); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs b/src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs new file mode 100644 index 00000000..a71b6327 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs @@ -0,0 +1,86 @@ +using System.Linq; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class CanonicalHashCalculatorTests +{ + private static readonly Advisory SampleAdvisory = new( + advisoryKey: "CVE-2024-0001", + title: "Sample advisory", + summary: "A sample summary", + language: "EN", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z"), + severity: "high", + exploitKnown: true, + aliases: new[] { "GHSA-xyz", "CVE-2024-0001" }, + references: new[] + { + new AdvisoryReference("https://example.com/advisory", "external", "vendor", summary: null, provenance: AdvisoryProvenance.Empty), + new AdvisoryReference("https://example.com/blog", "article", "blog", summary: null, provenance: AdvisoryProvenance.Empty), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/sample@1.0.0", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange("semver", "1.0.0", "1.2.0", null, null, AdvisoryProvenance.Empty), + new AffectedVersionRange("semver", "1.2.0", null, null, null, AdvisoryProvenance.Empty), + }, + statuses: Array.Empty(), + provenance: new[] { AdvisoryProvenance.Empty }) + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", 9.8, "critical", AdvisoryProvenance.Empty) + }, + provenance: new[] { AdvisoryProvenance.Empty }); + + [Fact] + public void ComputeHash_ReturnsDeterministicValue() + { + var calculator = new CanonicalHashCalculator(); + var first = calculator.ComputeHash(SampleAdvisory); + var second = calculator.ComputeHash(SampleAdvisory); + + Assert.Equal(first, second); + } + + [Fact] + public void ComputeHash_IgnoresOrderingDifferences() + { + var calculator = new CanonicalHashCalculator(); + + var reordered = new Advisory( + SampleAdvisory.AdvisoryKey, + SampleAdvisory.Title, + SampleAdvisory.Summary, + SampleAdvisory.Language, + SampleAdvisory.Published, + SampleAdvisory.Modified, + SampleAdvisory.Severity, + SampleAdvisory.ExploitKnown, + aliases: SampleAdvisory.Aliases.Reverse().ToArray(), + references: SampleAdvisory.References.Reverse().ToArray(), + affectedPackages: SampleAdvisory.AffectedPackages.Reverse().ToArray(), + cvssMetrics: SampleAdvisory.CvssMetrics.Reverse().ToArray(), + provenance: SampleAdvisory.Provenance.Reverse().ToArray()); + + var originalHash = calculator.ComputeHash(SampleAdvisory); + var reorderedHash = calculator.ComputeHash(reordered); + + Assert.Equal(originalHash, reorderedHash); + } + + [Fact] + public void ComputeHash_NullReturnsEmpty() + { + var calculator = new CanonicalHashCalculator(); + Assert.Empty(calculator.ComputeHash(null)); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs b/src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs new file mode 100644 index 00000000..a35c4489 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs @@ -0,0 +1,84 @@ +using StellaOps.Feedser.Merge.Comparers; +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class DebianEvrComparerTests +{ + [Theory] + [InlineData("1:1.2.3-1", 1, "1.2.3", "1")] + [InlineData("1.2.3-1", 0, "1.2.3", "1")] + [InlineData("2:4.5", 2, "4.5", "")] + [InlineData("abc", 0, "abc", "")] + public void TryParse_ReturnsComponents(string input, int expectedEpoch, string expectedVersion, string expectedRevision) + { + var success = DebianEvr.TryParse(input, out var evr); + + Assert.True(success); + Assert.NotNull(evr); + Assert.Equal(expectedEpoch, evr!.Epoch); + Assert.Equal(expectedVersion, evr.Version); + Assert.Equal(expectedRevision, evr.Revision); + Assert.Equal(input, evr.Original); + } + + [Theory] + [InlineData("")] + [InlineData(":1.0-1")] + [InlineData("1:")] + public void TryParse_InvalidInputs_ReturnFalse(string input) + { + var success = DebianEvr.TryParse(input, out var evr); + + Assert.False(success); + Assert.Null(evr); + } + + [Fact] + public void Compare_PrefersHigherEpoch() + { + var lower = "0:2.0-1"; + var higher = "1:1.0-1"; + + Assert.True(DebianEvrComparer.Instance.Compare(higher, lower) > 0); + } + + [Fact] + public void Compare_UsesVersionOrdering() + { + var lower = "0:1.2.3-1"; + var higher = "0:1.10.0-1"; + + Assert.True(DebianEvrComparer.Instance.Compare(higher, lower) > 0); + } + + [Fact] + public void Compare_TildeRanksEarlier() + { + var prerelease = "0:1.0~beta1-1"; + var stable = "0:1.0-1"; + + Assert.True(DebianEvrComparer.Instance.Compare(prerelease, stable) < 0); + } + + [Fact] + public void Compare_RevisionBreaksTies() + { + var first = "0:1.0-1"; + var second = "0:1.0-2"; + + Assert.True(DebianEvrComparer.Instance.Compare(second, first) > 0); + } + + [Fact] + public void Compare_FallsBackToOrdinalForInvalid() + { + var left = "not-an-evr"; + var right = "also-not"; + + var expected = Math.Sign(string.CompareOrdinal(left, right)); + var actual = Math.Sign(DebianEvrComparer.Instance.Compare(left, right)); + + Assert.Equal(expected, actual); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs b/src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs new file mode 100644 index 00000000..ff7c2f2c --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs @@ -0,0 +1,85 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class MergeEventWriterTests +{ + [Fact] + public async Task AppendAsync_WritesRecordWithComputedHashes() + { + var store = new InMemoryMergeEventStore(); + var calculator = new CanonicalHashCalculator(); + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2024-05-01T00:00:00Z")); + var writer = new MergeEventWriter(store, calculator, timeProvider, NullLogger.Instance); + + var before = CreateAdvisory("CVE-2024-0001", "Initial"); + var after = CreateAdvisory("CVE-2024-0001", "Sample", summary: "Updated"); + + var documentIds = new[] { Guid.NewGuid(), Guid.NewGuid() }; + var record = await writer.AppendAsync("CVE-2024-0001", before, after, documentIds, CancellationToken.None); + + Assert.NotEqual(Guid.Empty, record.Id); + Assert.Equal("CVE-2024-0001", record.AdvisoryKey); + Assert.True(record.AfterHash.Length > 0); + Assert.Equal(timeProvider.GetUtcNow(), record.MergedAt); + Assert.Equal(documentIds, record.InputDocumentIds); + Assert.NotNull(store.LastRecord); + Assert.Same(store.LastRecord, record); + } + + [Fact] + public async Task AppendAsync_NullBeforeUsesEmptyHash() + { + var store = new InMemoryMergeEventStore(); + var calculator = new CanonicalHashCalculator(); + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2024-05-01T00:00:00Z")); + var writer = new MergeEventWriter(store, calculator, timeProvider, NullLogger.Instance); + + var after = CreateAdvisory("CVE-2024-0002", "Changed"); + + var record = await writer.AppendAsync("CVE-2024-0002", null, after, Array.Empty(), CancellationToken.None); + + Assert.Empty(record.BeforeHash); + Assert.True(record.AfterHash.Length > 0); + } + + + private static Advisory CreateAdvisory(string advisoryKey, string title, string? summary = null) + { + return new Advisory( + advisoryKey, + title, + summary, + language: "en", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z"), + severity: "medium", + exploitKnown: false, + aliases: new[] { advisoryKey }, + references: new[] + { + new AdvisoryReference("https://example.com/" + advisoryKey.ToLowerInvariant(), "external", "vendor", summary: null, provenance: AdvisoryProvenance.Empty) + }, + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + } + + private sealed class InMemoryMergeEventStore : IMergeEventStore + { + public MergeEventRecord? LastRecord { get; private set; } + + public Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken) + { + LastRecord = record; + return Task.CompletedTask; + } + + public Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken) + => Task.FromResult>(Array.Empty()); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs b/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs new file mode 100644 index 00000000..5297ec41 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs @@ -0,0 +1,211 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Driver; +using StellaOps.Feedser.Merge.Services; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Merge.Tests; + +[Collection("mongo-fixture")] +public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private MergeEventStore? _mergeEventStore; + private MergeEventWriter? _mergeEventWriter; + private AdvisoryPrecedenceMerger? _merger; + private FakeTimeProvider? _timeProvider; + + public MergePrecedenceIntegrationTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task MergePipeline_PsirtOverridesNvd_AndKevOnlyTogglesExploitKnown() + { + await EnsureInitializedAsync(); + + var merger = _merger!; + var writer = _mergeEventWriter!; + var store = _mergeEventStore!; + var timeProvider = _timeProvider!; + + var expectedTimestamp = timeProvider.GetUtcNow(); + + var nvd = CreateNvdBaseline(); + var vendor = CreateVendorOverride(); + var kev = CreateKevSignal(); + + var merged = merger.Merge(new[] { nvd, vendor, kev }); + + Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); + Assert.Equal("Vendor Security Advisory", merged.Title); + Assert.Equal("Critical impact on supported platforms.", merged.Summary); + Assert.Equal("critical", merged.Severity); + Assert.True(merged.ExploitKnown); + + var affected = Assert.Single(merged.AffectedPackages); + Assert.Empty(affected.VersionRanges); + Assert.Contains(affected.Statuses, status => status.Status == "known_affected" && status.Provenance.Source == "vendor"); + + var mergeProvenance = Assert.Single(merged.Provenance, p => p.Source == "merge"); + Assert.Equal("precedence", mergeProvenance.Kind); + Assert.Equal(expectedTimestamp, mergeProvenance.RecordedAt); + Assert.Contains("vendor", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + Assert.Contains("kev", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); + + var inputDocumentIds = new[] { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() }; + var record = await writer.AppendAsync(merged.AdvisoryKey, nvd, merged, inputDocumentIds, CancellationToken.None); + + Assert.Equal(expectedTimestamp, record.MergedAt); + Assert.Equal(inputDocumentIds, record.InputDocumentIds); + Assert.NotEqual(record.BeforeHash, record.AfterHash); + + var records = await store.GetRecentAsync(merged.AdvisoryKey, 5, CancellationToken.None); + var persisted = Assert.Single(records); + Assert.Equal(record.Id, persisted.Id); + Assert.Equal(merged.AdvisoryKey, persisted.AdvisoryKey); + Assert.True(persisted.AfterHash.Length > 0); + Assert.True(persisted.BeforeHash.Length > 0); + } + + public async Task InitializeAsync() + { + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero)) + { + AutoAdvanceAmount = TimeSpan.Zero, + }; + _merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), _timeProvider); + _mergeEventStore = new MergeEventStore(_fixture.Database, NullLogger.Instance); + _mergeEventWriter = new MergeEventWriter(_mergeEventStore, new CanonicalHashCalculator(), _timeProvider, NullLogger.Instance); + await DropMergeCollectionAsync(); + } + + public Task DisposeAsync() => Task.CompletedTask; + + private async Task EnsureInitializedAsync() + { + if (_mergeEventWriter is null) + { + await InitializeAsync(); + } + } + + private async Task DropMergeCollectionAsync() + { + try + { + await _fixture.Database.DropCollectionAsync(MongoStorageDefaults.Collections.MergeEvent); + } + catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound" || ex.Message.Contains("ns not found", StringComparison.OrdinalIgnoreCase)) + { + // Collection has not been created yet – safe to ignore. + } + } + + private static Advisory CreateNvdBaseline() + { + var provenance = new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-1000", DateTimeOffset.Parse("2025-02-10T00:00:00Z")); + return new Advisory( + "CVE-2025-1000", + "CVE-2025-1000", + "Baseline description from NVD.", + "en", + DateTimeOffset.Parse("2025-02-05T00:00:00Z"), + DateTimeOffset.Parse("2025-02-10T12:00:00Z"), + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000" }, + references: new[] + { + new AdvisoryReference("https://nvd.nist.gov/vuln/detail/CVE-2025-1000", "advisory", "nvd", "NVD reference", provenance), + }, + affectedPackages: new[] + { + new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:o:vendor:product:1.0:*:*:*:*:*:*:*", + "vendor-os", + new[] + { + new AffectedVersionRange( + rangeKind: "cpe", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: "<=1.0", + provenance: provenance) + }, + Array.Empty(), + new[] { provenance }) + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", 9.8, "critical", provenance) + }, + provenance: new[] { provenance }); + } + + private static Advisory CreateVendorOverride() + { + var provenance = new AdvisoryProvenance("vendor", "psirt", "VSA-2025-1000", DateTimeOffset.Parse("2025-02-11T00:00:00Z")); + return new Advisory( + "CVE-2025-1000", + "Vendor Security Advisory", + "Critical impact on supported platforms.", + "en", + DateTimeOffset.Parse("2025-02-06T00:00:00Z"), + DateTimeOffset.Parse("2025-02-11T06:00:00Z"), + "critical", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000", "VSA-2025-1000" }, + references: new[] + { + new AdvisoryReference("https://vendor.example/advisories/VSA-2025-1000", "advisory", "vendor", "Vendor advisory", provenance), + }, + affectedPackages: new[] + { + new AffectedPackage( + AffectedPackageTypes.Cpe, + "cpe:2.3:o:vendor:product:1.0:*:*:*:*:*:*:*", + "vendor-os", + Array.Empty(), + new[] + { + new AffectedPackageStatus("known_affected", provenance) + }, + new[] { provenance }) + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", 10.0, "critical", provenance) + }, + provenance: new[] { provenance }); + } + + private static Advisory CreateKevSignal() + { + var provenance = new AdvisoryProvenance("kev", "catalog", "CVE-2025-1000", DateTimeOffset.Parse("2025-02-12T00:00:00Z")); + return new Advisory( + "CVE-2025-1000", + "Known Exploited Vulnerability", + null, + null, + published: null, + modified: null, + severity: null, + exploitKnown: true, + aliases: new[] { "KEV-CVE-2025-1000" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs b/src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs new file mode 100644 index 00000000..2531672a --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; + +namespace StellaOps.Feedser.Merge.Tests; + +internal sealed class MetricCollector : IDisposable +{ + private readonly MeterListener _listener; + private readonly List _measurements = new(); + + public MetricCollector(string meterName) + { + if (string.IsNullOrWhiteSpace(meterName)) + { + throw new ArgumentException("Meter name is required", nameof(meterName)); + } + + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == meterName) + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + + _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var tagArray = new KeyValuePair[tags.Length]; + for (var i = 0; i < tags.Length; i++) + { + tagArray[i] = tags[i]; + } + + _measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagArray)); + }); + + _listener.Start(); + } + + public IReadOnlyList Measurements => _measurements; + + public void Dispose() + { + _listener.Dispose(); + } + + internal sealed record MetricMeasurement( + string Name, + long Value, + IReadOnlyList> Tags); +} diff --git a/src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs b/src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs new file mode 100644 index 00000000..96418dd3 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs @@ -0,0 +1,108 @@ +using StellaOps.Feedser.Merge.Comparers; +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class NevraComparerTests +{ + [Theory] + [InlineData("kernel-1:4.18.0-348.7.1.el8_5.x86_64", "kernel", 1, "4.18.0", "348.7.1.el8_5", "x86_64")] + [InlineData("bash-5.1.8-2.fc35.x86_64", "bash", 0, "5.1.8", "2.fc35", "x86_64")] + [InlineData("openssl-libs-1:1.1.1k-7.el8", "openssl-libs", 1, "1.1.1k", "7.el8", null)] + [InlineData("java-11-openjdk-1:11.0.23.0.9-2.el9_4.ppc64le", "java-11-openjdk", 1, "11.0.23.0.9", "2.el9_4", "ppc64le")] + [InlineData("bash-0:5.2.15-3.el9_4.arm64", "bash", 0, "5.2.15", "3.el9_4", "arm64")] + [InlineData("podman-3:4.9.3-1.el9.x86_64", "podman", 3, "4.9.3", "1.el9", "x86_64")] + public void TryParse_ReturnsExpectedComponents(string input, string expectedName, int expectedEpoch, string expectedVersion, string expectedRelease, string? expectedArch) + { + var success = Nevra.TryParse(input, out var nevra); + + Assert.True(success); + Assert.NotNull(nevra); + Assert.Equal(expectedName, nevra!.Name); + Assert.Equal(expectedEpoch, nevra.Epoch); + Assert.Equal(expectedVersion, nevra.Version); + Assert.Equal(expectedRelease, nevra.Release); + Assert.Equal(expectedArch, nevra.Architecture); + Assert.Equal(input, nevra.Original); + } + + [Theory] + [InlineData("")] + [InlineData("kernel4.18.0-80.el8")] + [InlineData("kernel-4.18.0")] + public void TryParse_InvalidInputs_ReturnFalse(string input) + { + var success = Nevra.TryParse(input, out var nevra); + + Assert.False(success); + Assert.Null(nevra); + } + + [Fact] + public void TryParse_TrimsWhitespace() + { + var success = Nevra.TryParse(" kernel-0:4.18.0-80.el8.x86_64 ", out var nevra); + + Assert.True(success); + Assert.NotNull(nevra); + Assert.Equal("kernel", nevra!.Name); + Assert.Equal("4.18.0", nevra.Version); + } + + [Fact] + public void Compare_PrefersHigherEpoch() + { + var older = "kernel-0:4.18.0-348.7.1.el8_5.x86_64"; + var newer = "kernel-1:4.18.0-348.7.1.el8_5.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(newer, older) > 0); + Assert.True(NevraComparer.Instance.Compare(older, newer) < 0); + } + + [Fact] + public void Compare_UsesRpmVersionOrdering() + { + var lower = "kernel-0:4.18.0-80.el8.x86_64"; + var higher = "kernel-0:4.18.11-80.el8.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(higher, lower) > 0); + } + + [Fact] + public void Compare_UsesReleaseOrdering() + { + var el8 = "bash-0:5.1.0-1.el8.x86_64"; + var el9 = "bash-0:5.1.0-1.el9.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(el9, el8) > 0); + } + + [Fact] + public void Compare_TildeRanksEarlier() + { + var prerelease = "bash-0:5.1.0~beta-1.fc34.x86_64"; + var stable = "bash-0:5.1.0-1.fc34.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(prerelease, stable) < 0); + } + + [Fact] + public void Compare_ConsidersArchitecture() + { + var noarch = "pkg-0:1.0-1.noarch"; + var arch = "pkg-0:1.0-1.x86_64"; + + Assert.True(NevraComparer.Instance.Compare(noarch, arch) < 0); + } + + [Fact] + public void Compare_FallsBackToOrdinalForInvalid() + { + var left = "not-a-nevra"; + var right = "also-not"; + + var expected = Math.Sign(string.CompareOrdinal(left, right)); + var actual = Math.Sign(NevraComparer.Instance.Compare(left, right)); + Assert.Equal(expected, actual); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs new file mode 100644 index 00000000..8d6450d4 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs @@ -0,0 +1,67 @@ +using StellaOps.Feedser.Merge.Comparers; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class SemanticVersionRangeResolverTests +{ + [Theory] + [InlineData("1.2.3", true)] + [InlineData("1.2.3-beta.1", true)] + [InlineData("invalid", false)] + [InlineData(null, false)] + public void TryParse_ReturnsExpected(string? input, bool expected) + { + var success = SemanticVersionRangeResolver.TryParse(input, out var version); + + Assert.Equal(expected, success); + Assert.Equal(expected, version is not null); + } + + [Fact] + public void Compare_ParsesSemanticVersions() + { + Assert.True(SemanticVersionRangeResolver.Compare("1.2.3", "1.2.2") > 0); + Assert.True(SemanticVersionRangeResolver.Compare("1.2.3-beta", "1.2.3") < 0); + } + + [Fact] + public void Compare_UsesOrdinalFallbackForInvalid() + { + var left = "zzz"; + var right = "aaa"; + var expected = Math.Sign(string.CompareOrdinal(left, right)); + var actual = Math.Sign(SemanticVersionRangeResolver.Compare(left, right)); + + Assert.Equal(expected, actual); + } + + [Fact] + public void ResolveWindows_WithFixedVersion_ComputesExclusiveUpper() + { + var (introduced, exclusive, inclusive) = SemanticVersionRangeResolver.ResolveWindows("1.0.0", "1.2.0", null); + + Assert.Equal(SemanticVersionRangeResolver.Parse("1.0.0"), introduced); + Assert.Equal(SemanticVersionRangeResolver.Parse("1.2.0"), exclusive); + Assert.Null(inclusive); + } + + [Fact] + public void ResolveWindows_WithLastAffectedOnly_ComputesInclusiveAndExclusive() + { + var (introduced, exclusive, inclusive) = SemanticVersionRangeResolver.ResolveWindows("1.0.0", null, "1.1.5"); + + Assert.Equal(SemanticVersionRangeResolver.Parse("1.0.0"), introduced); + Assert.Equal(SemanticVersionRangeResolver.Parse("1.1.6"), exclusive); + Assert.Equal(SemanticVersionRangeResolver.Parse("1.1.5"), inclusive); + } + + [Fact] + public void ResolveWindows_WithNeither_ReturnsNullBounds() + { + var (introduced, exclusive, inclusive) = SemanticVersionRangeResolver.ResolveWindows(null, null, null); + + Assert.Null(introduced); + Assert.Null(exclusive); + Assert.Null(inclusive); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj b/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj new file mode 100644 index 00000000..209ec7c7 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Merge.Tests/TestLogger.cs b/src/StellaOps.Feedser.Merge.Tests/TestLogger.cs new file mode 100644 index 00000000..aa250a3a --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/TestLogger.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Merge.Tests; + +internal sealed class TestLogger : ILogger +{ + private static readonly IDisposable NoopScope = new DisposableScope(); + + public List Entries { get; } = new(); + + public IDisposable BeginScope(TState state) + where TState : notnull + => NoopScope; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (formatter is null) + { + throw new ArgumentNullException(nameof(formatter)); + } + + IReadOnlyList>? structuredState = null; + if (state is IReadOnlyList> list) + { + structuredState = list.ToArray(); + } + else if (state is IEnumerable> enumerable) + { + structuredState = enumerable.ToArray(); + } + + Entries.Add(new LogEntry(logLevel, eventId, formatter(state, exception), structuredState)); + } + + internal sealed record LogEntry( + LogLevel Level, + EventId EventId, + string Message, + IReadOnlyList>? StructuredState); + + private sealed class DisposableScope : IDisposable + { + public void Dispose() + { + } + } +} diff --git a/src/StellaOps.Feedser.Merge/AGENTS.md b/src/StellaOps.Feedser.Merge/AGENTS.md new file mode 100644 index 00000000..6c45a964 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/AGENTS.md @@ -0,0 +1,33 @@ +# AGENTS +## Role +Deterministic merge and reconciliation engine; builds identity graph via aliases; applies precedence (PSIRT/OVAL > NVD; KEV flag only; regional feeds enrich); produces canonical advisory JSON and merge_event audit trail. +## Scope +- Identity: resolve advisory_key (prefer CVE, else PSIRT/Distro/JVN/BDU/GHSA/ICSA); unify aliases; detect collisions. +- Precedence: override rules for affected ranges (vendor PSIRT/OVAL over registry), enrichment-only feeds (CERTs/JVN/RU-CERT), KEV toggles exploitKnown only. +- Range comparers: RPM NEVRA comparer (epoch:version-release), Debian EVR comparer, SemVer range resolver; platform-aware selection. +- Merge algorithm: stable ordering, pure functions, idempotence; compute beforeHash/afterHash over canonical form; write merge_event. +- Conflict reporting: counters and logs for identity conflicts, reference merges, range overrides. +## Participants +- Storage.Mongo (reads raw mapped advisories, writes merged docs plus merge_event). +- Models (canonical types). +- Exporters (consume merged canonical). +- Core/WebService (jobs: merge:run, maybe per-kind). +## Interfaces & contracts +- AdvisoryMergeService.MergeAsync(ids or byKind): returns summary {processed, merged, overrides, conflicts}. +- Precedence table configurable but with sane defaults: RedHat/Ubuntu/Debian/SUSE > Vendor PSIRT > GHSA/OSV > NVD; CERTs enrich; KEV sets flags. +- Range selection uses comparers: NevraComparer, DebEvrComparer, SemVerRange; deterministic tie-breakers. +- Provenance propagation merges unique entries; references deduped by (url, type). + +## Configuration +- Precedence overrides bind via `feedser:merge:precedence:ranks` (dictionary of `source` → `rank`, lower wins). Absent entries fall back to defaults. +- Operator workflow: update `etc/feedser.yaml` or environment variables, restart merge job; overrides surface in metrics/logs as `AdvisoryOverride` entries. +## In/Out of scope +In: merge logic, precedence policy, hashing, event records, comparers. +Out: fetching/parsing, exporter packaging, signing. +## Observability & security expectations +- Metrics: merge.delta.count, merge.identity.conflicts, merge.range.overrides, merge.duration_ms. +- Logs: decisions (why replaced), keys involved, hashes; avoid dumping large blobs; redact secrets (none expected). +## Tests +- Author and review coverage in `../StellaOps.Feedser.Merge.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Merge/Class1.cs b/src/StellaOps.Feedser.Merge/Class1.cs new file mode 100644 index 00000000..1efda537 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Class1.cs @@ -0,0 +1 @@ +// Intentionally left blank; types moved into dedicated files. diff --git a/src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs b/src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs new file mode 100644 index 00000000..c4ebb786 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs @@ -0,0 +1,232 @@ +namespace StellaOps.Feedser.Merge.Comparers; + +using System; +using StellaOps.Feedser.Normalization.Distro; + +public sealed class DebianEvrComparer : IComparer, IComparer +{ + public static DebianEvrComparer Instance { get; } = new(); + + private DebianEvrComparer() + { + } + + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var xParsed = DebianEvr.TryParse(x, out var xEvr); + var yParsed = DebianEvr.TryParse(y, out var yEvr); + + if (xParsed && yParsed) + { + return Compare(xEvr, yEvr); + } + + if (xParsed) + { + return 1; + } + + if (yParsed) + { + return -1; + } + + return string.Compare(x, y, StringComparison.Ordinal); + } + + public int Compare(DebianEvr? x, DebianEvr? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var compare = x.Epoch.CompareTo(y.Epoch); + if (compare != 0) + { + return compare; + } + + compare = CompareSegment(x.Version, y.Version); + if (compare != 0) + { + return compare; + } + + compare = CompareSegment(x.Revision, y.Revision); + if (compare != 0) + { + return compare; + } + + return string.Compare(x.Original, y.Original, StringComparison.Ordinal); + } + + private static int CompareSegment(string left, string right) + { + var i = 0; + var j = 0; + + while (i < left.Length || j < right.Length) + { + while (i < left.Length && !IsAlphaNumeric(left[i]) && left[i] != '~') + { + i++; + } + + while (j < right.Length && !IsAlphaNumeric(right[j]) && right[j] != '~') + { + j++; + } + + var leftChar = i < left.Length ? left[i] : '\0'; + var rightChar = j < right.Length ? right[j] : '\0'; + + if (leftChar == '~' || rightChar == '~') + { + if (leftChar != rightChar) + { + return leftChar == '~' ? -1 : 1; + } + + i += leftChar == '~' ? 1 : 0; + j += rightChar == '~' ? 1 : 0; + continue; + } + + var leftIsDigit = char.IsDigit(leftChar); + var rightIsDigit = char.IsDigit(rightChar); + + if (leftIsDigit && rightIsDigit) + { + var leftStart = i; + while (i < left.Length && char.IsDigit(left[i])) + { + i++; + } + + var rightStart = j; + while (j < right.Length && char.IsDigit(right[j])) + { + j++; + } + + var leftTrimmed = leftStart; + while (leftTrimmed < i && left[leftTrimmed] == '0') + { + leftTrimmed++; + } + + var rightTrimmed = rightStart; + while (rightTrimmed < j && right[rightTrimmed] == '0') + { + rightTrimmed++; + } + + var leftLength = i - leftTrimmed; + var rightLength = j - rightTrimmed; + + if (leftLength != rightLength) + { + return leftLength.CompareTo(rightLength); + } + + var comparison = left.AsSpan(leftTrimmed, leftLength) + .CompareTo(right.AsSpan(rightTrimmed, rightLength), StringComparison.Ordinal); + if (comparison != 0) + { + return comparison; + } + + continue; + } + + if (leftIsDigit) + { + return 1; + } + + if (rightIsDigit) + { + return -1; + } + + var leftOrder = CharOrder(leftChar); + var rightOrder = CharOrder(rightChar); + + var orderComparison = leftOrder.CompareTo(rightOrder); + if (orderComparison != 0) + { + return orderComparison; + } + + if (leftChar != rightChar) + { + return leftChar.CompareTo(rightChar); + } + + if (leftChar == '\0') + { + return 0; + } + + i++; + j++; + } + + return 0; + } + + private static bool IsAlphaNumeric(char value) + => char.IsLetterOrDigit(value); + + private static int CharOrder(char value) + { + if (value == '\0') + { + return 0; + } + + if (value == '~') + { + return -1; + } + + if (char.IsDigit(value)) + { + return 0; + } + + if (char.IsLetter(value)) + { + return value; + } + + return value + 256; + } +} diff --git a/src/StellaOps.Feedser.Merge/Comparers/Nevra.cs b/src/StellaOps.Feedser.Merge/Comparers/Nevra.cs new file mode 100644 index 00000000..0870b20f --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Comparers/Nevra.cs @@ -0,0 +1,264 @@ +namespace StellaOps.Feedser.Merge.Comparers; + +using System; +using StellaOps.Feedser.Normalization.Distro; + +public sealed class NevraComparer : IComparer, IComparer +{ + public static NevraComparer Instance { get; } = new(); + + private NevraComparer() + { + } + + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var xParsed = Nevra.TryParse(x, out var xNevra); + var yParsed = Nevra.TryParse(y, out var yNevra); + + if (xParsed && yParsed) + { + return Compare(xNevra, yNevra); + } + + if (xParsed) + { + return 1; + } + + if (yParsed) + { + return -1; + } + + return string.Compare(x, y, StringComparison.Ordinal); + } + + public int Compare(Nevra? x, Nevra? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var compare = string.Compare(x.Name, y.Name, StringComparison.Ordinal); + if (compare != 0) + { + return compare; + } + + compare = string.Compare(x.Architecture ?? string.Empty, y.Architecture ?? string.Empty, StringComparison.Ordinal); + if (compare != 0) + { + return compare; + } + + compare = x.Epoch.CompareTo(y.Epoch); + if (compare != 0) + { + return compare; + } + + compare = RpmVersionComparer.Compare(x.Version, y.Version); + if (compare != 0) + { + return compare; + } + + compare = RpmVersionComparer.Compare(x.Release, y.Release); + if (compare != 0) + { + return compare; + } + + return string.Compare(x.Original, y.Original, StringComparison.Ordinal); + } +} + +internal static class RpmVersionComparer +{ + public static int Compare(string? left, string? right) + { + left ??= string.Empty; + right ??= string.Empty; + + var i = 0; + var j = 0; + + while (true) + { + var leftHasTilde = SkipToNextSegment(left, ref i); + var rightHasTilde = SkipToNextSegment(right, ref j); + + if (leftHasTilde || rightHasTilde) + { + if (leftHasTilde && rightHasTilde) + { + continue; + } + + return leftHasTilde ? -1 : 1; + } + + var leftEnd = i >= left.Length; + var rightEnd = j >= right.Length; + if (leftEnd || rightEnd) + { + if (leftEnd && rightEnd) + { + return 0; + } + + return leftEnd ? -1 : 1; + } + + var leftDigit = char.IsDigit(left[i]); + var rightDigit = char.IsDigit(right[j]); + + if (leftDigit && !rightDigit) + { + return 1; + } + + if (!leftDigit && rightDigit) + { + return -1; + } + + int compare; + if (leftDigit) + { + compare = CompareNumericSegment(left, ref i, right, ref j); + } + else + { + compare = CompareAlphaSegment(left, ref i, right, ref j); + } + + if (compare != 0) + { + return compare; + } + } + } + + private static bool SkipToNextSegment(string value, ref int index) + { + var sawTilde = false; + while (index < value.Length) + { + var current = value[index]; + if (current == '~') + { + sawTilde = true; + index++; + break; + } + + if (char.IsLetterOrDigit(current)) + { + break; + } + + index++; + } + + return sawTilde; + } + + private static int CompareNumericSegment(string value, ref int index, string other, ref int otherIndex) + { + var start = index; + while (index < value.Length && char.IsDigit(value[index])) + { + index++; + } + + var otherStart = otherIndex; + while (otherIndex < other.Length && char.IsDigit(other[otherIndex])) + { + otherIndex++; + } + + var trimmedStart = start; + while (trimmedStart < index && value[trimmedStart] == '0') + { + trimmedStart++; + } + + var otherTrimmedStart = otherStart; + while (otherTrimmedStart < otherIndex && other[otherTrimmedStart] == '0') + { + otherTrimmedStart++; + } + + var length = index - trimmedStart; + var otherLength = otherIndex - otherTrimmedStart; + + if (length != otherLength) + { + return length.CompareTo(otherLength); + } + + var comparison = value.AsSpan(trimmedStart, length) + .CompareTo(other.AsSpan(otherTrimmedStart, otherLength), StringComparison.Ordinal); + if (comparison != 0) + { + return comparison; + } + + return 0; + } + + private static int CompareAlphaSegment(string value, ref int index, string other, ref int otherIndex) + { + var start = index; + while (index < value.Length && char.IsLetter(value[index])) + { + index++; + } + + var otherStart = otherIndex; + while (otherIndex < other.Length && char.IsLetter(other[otherIndex])) + { + otherIndex++; + } + + var length = index - start; + var otherLength = otherIndex - otherStart; + + var comparison = value.AsSpan(start, length) + .CompareTo(other.AsSpan(otherStart, otherLength), StringComparison.Ordinal); + if (comparison != 0) + { + return comparison; + } + + return 0; + } +} diff --git a/src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs b/src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs new file mode 100644 index 00000000..c333ab2a --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs @@ -0,0 +1,73 @@ +namespace StellaOps.Feedser.Merge.Comparers; + +using System.Diagnostics.CodeAnalysis; +using Semver; + +/// +/// Provides helpers to interpret introduced/fixed/lastAffected SemVer ranges and compare versions. +/// +public static class SemanticVersionRangeResolver +{ + public static bool TryParse(string? value, [NotNullWhen(true)] out SemVersion? result) + => SemVersion.TryParse(value, SemVersionStyles.Any, out result); + + public static SemVersion Parse(string value) + => SemVersion.Parse(value, SemVersionStyles.Any); + + /// + /// Resolves the effective start and end versions using introduced/fixed/lastAffected semantics. + /// + public static (SemVersion? introduced, SemVersion? exclusiveUpperBound, SemVersion? inclusiveUpperBound) ResolveWindows( + string? introduced, + string? fixedVersion, + string? lastAffected) + { + var introducedVersion = TryParse(introduced, out var parsedIntroduced) ? parsedIntroduced : null; + var fixedVersionParsed = TryParse(fixedVersion, out var parsedFixed) ? parsedFixed : null; + var lastAffectedVersion = TryParse(lastAffected, out var parsedLast) ? parsedLast : null; + + SemVersion? exclusiveUpper = null; + SemVersion? inclusiveUpper = null; + + if (fixedVersionParsed is not null) + { + exclusiveUpper = fixedVersionParsed; + } + else if (lastAffectedVersion is not null) + { + inclusiveUpper = lastAffectedVersion; + exclusiveUpper = NextPatch(lastAffectedVersion); + } + + return (introducedVersion, exclusiveUpper, inclusiveUpper); + } + + + public static int Compare(string? left, string? right) + { + var leftParsed = TryParse(left, out var leftSemver); + var rightParsed = TryParse(right, out var rightSemver); + + if (leftParsed && rightParsed) + { + return SemVersion.CompareSortOrder(leftSemver, rightSemver); + } + + if (leftParsed) + { + return 1; + } + + if (rightParsed) + { + return -1; + } + + return string.Compare(left, right, StringComparison.Ordinal); + } + + private static SemVersion NextPatch(SemVersion version) + { + return new SemVersion(version.Major, version.Minor, version.Patch + 1); + } +} diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs new file mode 100644 index 00000000..1326a49c --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Merge.Options; + +/// +/// Configurable precedence overrides for advisory sources. +/// +public sealed class AdvisoryPrecedenceOptions +{ + /// + /// Mapping of provenance source identifiers to precedence ranks. Lower numbers take precedence. + /// + public IDictionary Ranks { get; init; } = new Dictionary(StringComparer.OrdinalIgnoreCase); +} diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs new file mode 100644 index 00000000..a5495af1 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Merge.Options; + +internal static class AdvisoryPrecedenceTable +{ + public static IReadOnlyDictionary Merge( + IReadOnlyDictionary defaults, + AdvisoryPrecedenceOptions? options) + { + if (defaults is null) + { + throw new ArgumentNullException(nameof(defaults)); + } + + if (options?.Ranks is null || options.Ranks.Count == 0) + { + return defaults; + } + + var merged = new Dictionary(defaults, StringComparer.OrdinalIgnoreCase); + foreach (var kvp in options.Ranks) + { + if (string.IsNullOrWhiteSpace(kvp.Key)) + { + continue; + } + + merged[kvp.Key.Trim()] = kvp.Value; + } + + return merged; + } +} diff --git a/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs b/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs new file mode 100644 index 00000000..348a4825 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs @@ -0,0 +1,357 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Merge.Options; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Services; + +/// +/// Merges canonical advisories emitted by different sources into a single precedence-resolved advisory. +/// +public sealed class AdvisoryPrecedenceMerger +{ + private static readonly IReadOnlyDictionary DefaultPrecedence = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["redhat"] = 0, + ["ubuntu"] = 0, + ["debian"] = 0, + ["suse"] = 0, + ["msrc"] = 1, + ["oracle"] = 1, + ["adobe"] = 1, + ["chromium"] = 1, + ["vendor"] = 1, + ["jvn"] = 2, + ["certfr"] = 2, + ["certin"] = 2, + ["ics-kaspersky"] = 2, + ["kev"] = 6, + ["nvd"] = 5, + }; + + private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); + private static readonly Counter OverridesCounter = MergeMeter.CreateCounter( + "feedser.merge.overrides", + unit: "count", + description: "Number of times lower-precedence advisories were overridden by higher-precedence sources."); + + private static readonly Action OverrideLogged = LoggerMessage.Define( + LogLevel.Information, + new EventId(1000, "AdvisoryOverride"), + "Advisory precedence override {@Override}"); + + private readonly AffectedPackagePrecedenceResolver _packageResolver; + private readonly IReadOnlyDictionary _precedence; + private readonly int _fallbackRank; + private readonly System.TimeProvider _timeProvider; + private readonly ILogger _logger; + + public AdvisoryPrecedenceMerger() + : this(new AffectedPackagePrecedenceResolver(), DefaultPrecedence, System.TimeProvider.System, NullLogger.Instance) + { + } + + public AdvisoryPrecedenceMerger(AffectedPackagePrecedenceResolver packageResolver, System.TimeProvider? timeProvider = null) + : this(packageResolver, DefaultPrecedence, timeProvider ?? System.TimeProvider.System, NullLogger.Instance) + { + } + + public AdvisoryPrecedenceMerger( + AffectedPackagePrecedenceResolver packageResolver, + IReadOnlyDictionary precedence, + System.TimeProvider timeProvider) + : this(packageResolver, precedence, timeProvider, NullLogger.Instance) + { + } + + public AdvisoryPrecedenceMerger( + AffectedPackagePrecedenceResolver packageResolver, + AdvisoryPrecedenceOptions? options, + System.TimeProvider timeProvider, + ILogger? logger = null) + : this( + EnsureResolver(packageResolver, options, out var precedence), + precedence, + timeProvider, + logger) + { + } + + public AdvisoryPrecedenceMerger( + AffectedPackagePrecedenceResolver packageResolver, + IReadOnlyDictionary precedence, + System.TimeProvider timeProvider, + ILogger? logger) + { + _packageResolver = packageResolver ?? throw new ArgumentNullException(nameof(packageResolver)); + _precedence = precedence ?? throw new ArgumentNullException(nameof(precedence)); + _fallbackRank = _precedence.Count == 0 ? 10 : _precedence.Values.Max() + 1; + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? NullLogger.Instance; + } + + public Advisory Merge(IEnumerable advisories) + { + if (advisories is null) + { + throw new ArgumentNullException(nameof(advisories)); + } + + var list = advisories.Where(static a => a is not null).ToList(); + if (list.Count == 0) + { + throw new ArgumentException("At least one advisory is required for merge.", nameof(advisories)); + } + + var advisoryKey = list[0].AdvisoryKey; + if (list.Any(advisory => !string.Equals(advisory.AdvisoryKey, advisoryKey, StringComparison.Ordinal))) + { + throw new ArgumentException("All advisories must share the same advisory key.", nameof(advisories)); + } + + var ordered = list + .Select(advisory => new AdvisoryEntry(advisory, GetRank(advisory))) + .OrderBy(entry => entry.Rank) + .ThenByDescending(entry => entry.Advisory.Provenance.Length) + .ToArray(); + + var primary = ordered[0].Advisory; + + var title = PickString(ordered, advisory => advisory.Title) ?? advisoryKey; + var summary = PickString(ordered, advisory => advisory.Summary); + var language = PickString(ordered, advisory => advisory.Language); + var severity = PickString(ordered, advisory => advisory.Severity); + + var aliases = ordered + .SelectMany(entry => entry.Advisory.Aliases) + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var references = ordered + .SelectMany(entry => entry.Advisory.References) + .Distinct() + .ToArray(); + + var affectedPackages = _packageResolver.Merge(ordered.SelectMany(entry => entry.Advisory.AffectedPackages)); + var cvssMetrics = ordered + .SelectMany(entry => entry.Advisory.CvssMetrics) + .Distinct() + .ToArray(); + + var published = PickDateTime(ordered, static advisory => advisory.Published); + var modified = PickDateTime(ordered, static advisory => advisory.Modified) ?? published; + + var provenance = ordered + .SelectMany(entry => entry.Advisory.Provenance) + .Distinct() + .ToList(); + + var precedenceTrace = ordered + .SelectMany(entry => entry.Sources) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static source => source, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var mergeProvenance = new AdvisoryProvenance( + source: "merge", + kind: "precedence", + value: string.Join("|", precedenceTrace), + recordedAt: _timeProvider.GetUtcNow()); + + provenance.Add(mergeProvenance); + + var exploitKnown = ordered.Any(entry => entry.Advisory.ExploitKnown); + + LogOverrides(advisoryKey, ordered); + + return new Advisory( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown, + aliases, + references, + affectedPackages, + cvssMetrics, + provenance); + } + + private string? PickString(IEnumerable ordered, Func selector) + { + foreach (var entry in ordered) + { + var value = selector(entry.Advisory); + if (!string.IsNullOrWhiteSpace(value)) + { + return value.Trim(); + } + } + + return null; + } + + private DateTimeOffset? PickDateTime(IEnumerable ordered, Func selector) + { + foreach (var entry in ordered) + { + var value = selector(entry.Advisory); + if (value.HasValue) + { + return value.Value.ToUniversalTime(); + } + } + + return null; + } + + private int GetRank(Advisory advisory) + { + var best = _fallbackRank; + foreach (var provenance in advisory.Provenance) + { + if (string.IsNullOrWhiteSpace(provenance.Source)) + { + continue; + } + + if (_precedence.TryGetValue(provenance.Source, out var rank) && rank < best) + { + best = rank; + } + } + + return best; + } + + private void LogOverrides(string advisoryKey, IReadOnlyList ordered) + { + if (ordered.Count <= 1) + { + return; + } + + var primary = ordered[0]; + var primaryRank = primary.Rank; + + for (var i = 1; i < ordered.Count; i++) + { + var candidate = ordered[i]; + if (candidate.Rank <= primaryRank) + { + continue; + } + + var tags = new KeyValuePair[] + { + new("primary_source", FormatSourceLabel(primary.Sources)), + new("suppressed_source", FormatSourceLabel(candidate.Sources)), + new("primary_rank", primaryRank), + new("suppressed_rank", candidate.Rank), + }; + + OverridesCounter.Add(1, tags); + + var audit = new MergeOverrideAudit( + advisoryKey, + primary.Sources, + primaryRank, + candidate.Sources, + candidate.Rank, + primary.Advisory.Aliases.Length, + candidate.Advisory.Aliases.Length, + primary.Advisory.Provenance.Length, + candidate.Advisory.Provenance.Length); + + OverrideLogged(_logger, audit, null); + } + } + + private readonly record struct AdvisoryEntry(Advisory Advisory, int Rank) + { + public IReadOnlyCollection Sources { get; } = Advisory.Provenance + .Select(static p => p.Source) + .Where(static source => !string.IsNullOrWhiteSpace(source)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static AffectedPackagePrecedenceResolver EnsureResolver( + AffectedPackagePrecedenceResolver? resolver, + AdvisoryPrecedenceOptions? options, + out IReadOnlyDictionary precedence) + { + precedence = AdvisoryPrecedenceTable.Merge(DefaultPrecedence, options); + + if (resolver is null) + { + return new AffectedPackagePrecedenceResolver(precedence); + } + + if (DictionaryEquals(resolver.Precedence, precedence)) + { + return resolver; + } + + return new AffectedPackagePrecedenceResolver(precedence); + } + + private static bool DictionaryEquals( + IReadOnlyDictionary left, + IReadOnlyDictionary right) + { + if (ReferenceEquals(left, right)) + { + return true; + } + + if (left.Count != right.Count) + { + return false; + } + + foreach (var (key, value) in left) + { + if (!right.TryGetValue(key, out var other) || other != value) + { + return false; + } + } + + return true; + } + + private static string FormatSourceLabel(IReadOnlyCollection sources) + { + if (sources.Count == 0) + { + return "unknown"; + } + + if (sources.Count == 1) + { + return sources.First(); + } + + return string.Join('|', sources.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).Take(3)); + } + + private readonly record struct MergeOverrideAudit( + string AdvisoryKey, + IReadOnlyCollection PrimarySources, + int PrimaryRank, + IReadOnlyCollection SuppressedSources, + int SuppressedRank, + int PrimaryAliasCount, + int SuppressedAliasCount, + int PrimaryProvenanceCount, + int SuppressedProvenanceCount); +} diff --git a/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs b/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs new file mode 100644 index 00000000..59028d8f --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Feedser.Merge.Options; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Services; + +/// +/// Applies source precedence rules to affected package sets so authoritative distro ranges override generic registry data. +/// +public sealed class AffectedPackagePrecedenceResolver +{ + private static readonly IReadOnlyDictionary DefaultPrecedence = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["redhat"] = 0, + ["ubuntu"] = 0, + ["debian"] = 0, + ["suse"] = 0, + ["msrc"] = 1, + ["oracle"] = 1, + ["adobe"] = 1, + ["chromium"] = 1, + ["vendor"] = 1, + ["nvd"] = 5, + }; + + private readonly IReadOnlyDictionary _precedence; + private readonly int _fallbackRank; + + public AffectedPackagePrecedenceResolver() + : this(DefaultPrecedence) + { + } + + public AffectedPackagePrecedenceResolver(AdvisoryPrecedenceOptions? options) + : this(AdvisoryPrecedenceTable.Merge(DefaultPrecedence, options)) + { + } + + public AffectedPackagePrecedenceResolver(IReadOnlyDictionary precedence) + { + _precedence = precedence ?? throw new ArgumentNullException(nameof(precedence)); + _fallbackRank = precedence.Count == 0 ? 10 : precedence.Values.Max() + 1; + } + + public IReadOnlyDictionary Precedence => _precedence; + + public IReadOnlyList Merge(IEnumerable packages) + { + ArgumentNullException.ThrowIfNull(packages); + + var grouped = packages + .Where(static pkg => pkg is not null) + .GroupBy(pkg => (pkg.Type, pkg.Identifier, pkg.Platform ?? string.Empty)); + + var resolved = new List(); + foreach (var group in grouped) + { + var ordered = group + .OrderBy(GetPrecedence) + .ThenByDescending(static pkg => pkg.Provenance.Length) + .ThenByDescending(static pkg => pkg.VersionRanges.Length); + + var primary = ordered.First(); + var provenance = ordered + .SelectMany(static pkg => pkg.Provenance) + .Where(static p => p is not null) + .Distinct() + .ToImmutableArray(); + + var statuses = ordered + .SelectMany(static pkg => pkg.Statuses) + .Distinct(AffectedPackageStatusEqualityComparer.Instance) + .ToImmutableArray(); + + var merged = new AffectedPackage( + primary.Type, + primary.Identifier, + string.IsNullOrWhiteSpace(primary.Platform) ? null : primary.Platform, + primary.VersionRanges, + statuses, + provenance); + + resolved.Add(merged); + } + + return resolved + .OrderBy(static pkg => pkg.Type, StringComparer.Ordinal) + .ThenBy(static pkg => pkg.Identifier, StringComparer.Ordinal) + .ThenBy(static pkg => pkg.Platform, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private int GetPrecedence(AffectedPackage package) + { + var bestRank = _fallbackRank; + foreach (var provenance in package.Provenance) + { + if (provenance is null || string.IsNullOrWhiteSpace(provenance.Source)) + { + continue; + } + + if (_precedence.TryGetValue(provenance.Source, out var rank) && rank < bestRank) + { + bestRank = rank; + } + } + + return bestRank; + } +} diff --git a/src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs b/src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs new file mode 100644 index 00000000..7fa8b96c --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs @@ -0,0 +1,25 @@ +namespace StellaOps.Feedser.Merge.Services; + +using System.Security.Cryptography; +using System.Text; +using StellaOps.Feedser.Models; + +/// +/// Computes deterministic hashes over canonical advisory JSON payloads. +/// +public sealed class CanonicalHashCalculator +{ + private static readonly UTF8Encoding Utf8NoBom = new(false); + + public byte[] ComputeHash(Advisory? advisory) + { + if (advisory is null) + { + return Array.Empty(); + } + + var canonical = CanonicalJsonSerializer.Serialize(CanonicalJsonSerializer.Normalize(advisory)); + var payload = Utf8NoBom.GetBytes(canonical); + return SHA256.HashData(payload); + } +} diff --git a/src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs b/src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs new file mode 100644 index 00000000..f02278dd --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs @@ -0,0 +1,70 @@ +namespace StellaOps.Feedser.Merge.Services; + +using System.Security.Cryptography; +using System.Linq; +using Microsoft.Extensions.Logging; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +/// +/// Persists merge events with canonical before/after hashes for auditability. +/// +public sealed class MergeEventWriter +{ + private readonly IMergeEventStore _mergeEventStore; + private readonly CanonicalHashCalculator _hashCalculator; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public MergeEventWriter( + IMergeEventStore mergeEventStore, + CanonicalHashCalculator hashCalculator, + TimeProvider timeProvider, + ILogger logger) + { + _mergeEventStore = mergeEventStore ?? throw new ArgumentNullException(nameof(mergeEventStore)); + _hashCalculator = hashCalculator ?? throw new ArgumentNullException(nameof(hashCalculator)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AppendAsync( + string advisoryKey, + Advisory? before, + Advisory after, + IReadOnlyList inputDocumentIds, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); + ArgumentNullException.ThrowIfNull(after); + + var beforeHash = _hashCalculator.ComputeHash(before); + var afterHash = _hashCalculator.ComputeHash(after); + var timestamp = _timeProvider.GetUtcNow(); + var documentIds = inputDocumentIds?.ToArray() ?? Array.Empty(); + + var record = new MergeEventRecord( + Guid.NewGuid(), + advisoryKey, + beforeHash, + afterHash, + timestamp, + documentIds); + + if (!CryptographicOperations.FixedTimeEquals(beforeHash, afterHash)) + { + _logger.LogInformation( + "Merge event for {AdvisoryKey} changed hash {BeforeHash} -> {AfterHash}", + advisoryKey, + Convert.ToHexString(beforeHash), + Convert.ToHexString(afterHash)); + } + else + { + _logger.LogInformation("Merge event for {AdvisoryKey} recorded without hash change", advisoryKey); + } + + await _mergeEventStore.AppendAsync(record, cancellationToken).ConfigureAwait(false); + return record; + } +} diff --git a/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj b/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj new file mode 100644 index 00000000..0c42770d --- /dev/null +++ b/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj @@ -0,0 +1,16 @@ + + + + + net10.0 + enable + enable + + + + + + + + + diff --git a/src/StellaOps.Feedser.Merge/TASKS.md b/src/StellaOps.Feedser.Merge/TASKS.md new file mode 100644 index 00000000..37122944 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|Deterministic key choice; cycle-safe.| +|Precedence policy engine|BE-Merge|Architecture|PSIRT/OVAL > NVD; CERTs enrich; KEV flag.| +|NEVRA comparer plus tests|BE-Merge (Distro WG)|Source.Distro fixtures|DONE – Added Nevra parser/comparer with tilde-aware rpm ordering and unit coverage.| +|Debian EVR comparer plus tests|BE-Merge (Distro WG)|Debian fixtures|DONE – DebianEvr comparer mirrors dpkg ordering with tilde/epoch handling and unit coverage.| +|SemVer range resolver plus tests|BE-Merge (OSS WG)|OSV/GHSA fixtures|DONE – SemanticVersionRangeResolver covers introduced/fixed/lastAffected semantics with SemVer ordering tests.| +|Canonical hash and merge_event writer|BE-Merge|Models, Storage.Mongo|DONE – Hash calculator + MergeEventWriter compute canonical SHA-256 digests and persist merge events.| +|Conflict detection and metrics|BE-Merge|Core|Counters; structured logs; traces.| +|End-to-end determinism test|QA|Merge, key connectors|Same inputs -> same hashes.| +|Override audit logging|BE-Merge|Observability|DONE – override audits now emit structured logs plus bounded-tag metrics suitable for prod telemetry.| +|Configurable precedence table|BE-Merge|Architecture|DONE – precedence options bind via feedser:merge:precedence:ranks with docs/tests covering operator workflow.| diff --git a/src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs b/src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs new file mode 100644 index 00000000..653f6acd --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs @@ -0,0 +1,62 @@ +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class AdvisoryTests +{ + [Fact] + public void CanonicalizesAliasesAndReferences() + { + var advisory = new Advisory( + advisoryKey: "TEST-123", + title: "Sample Advisory", + summary: " summary with spaces ", + language: "EN", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z"), + severity: "CRITICAL", + exploitKnown: true, + aliases: new[] { " CVE-2024-0001", "GHSA-aaaa", "cve-2024-0001" }, + references: new[] + { + new AdvisoryReference("https://example.com/b", "patch", null, null, AdvisoryProvenance.Empty), + new AdvisoryReference("https://example.com/a", null, null, null, AdvisoryProvenance.Empty), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/sample", + platform: "node", + versionRanges: new[] + { + new AffectedVersionRange("semver", "1.0.0", "1.0.1", null, null, AdvisoryProvenance.Empty), + new AffectedVersionRange("semver", "1.0.0", "1.0.1", null, null, AdvisoryProvenance.Empty), + new AffectedVersionRange("semver", "0.9.0", null, "0.9.9", null, AdvisoryProvenance.Empty), + }, + statuses: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "map", "", DateTimeOffset.Parse("2024-01-01T00:00:00Z")), + new AdvisoryProvenance("vendor", "map", "", DateTimeOffset.Parse("2024-01-02T00:00:00Z")), + }) + }, + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "map", "", DateTimeOffset.Parse("2024-01-01T00:00:00Z")), + new AdvisoryProvenance("vendor", "map", "", DateTimeOffset.Parse("2024-01-02T00:00:00Z")), + }); + + Assert.Equal(new[] { "CVE-2024-0001", "GHSA-aaaa", "cve-2024-0001" }, advisory.Aliases); + Assert.Equal(new[] { "https://example.com/a", "https://example.com/b" }, advisory.References.Select(r => r.Url)); + Assert.Equal( + new[] + { + "semver|0.9.0||0.9.9|", + "semver|1.0.0|1.0.1||", + }, + advisory.AffectedPackages.Single().VersionRanges.Select(r => r.CreateDeterministicKey())); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs b/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs new file mode 100644 index 00000000..e90401cf --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs @@ -0,0 +1,28 @@ +using System; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class AffectedPackageStatusTests +{ + [Theory] + [InlineData("Known_Affected", AffectedPackageStatusCatalog.KnownAffected)] + [InlineData("KNOWN-NOT-AFFECTED", AffectedPackageStatusCatalog.KnownNotAffected)] + [InlineData("Under Investigation", AffectedPackageStatusCatalog.UnderInvestigation)] + [InlineData("Fixed", AffectedPackageStatusCatalog.Fixed)] + public void Constructor_NormalizesStatus(string input, string expected) + { + var provenance = new AdvisoryProvenance("test", "status", "value", DateTimeOffset.UtcNow); + var status = new AffectedPackageStatus(input, provenance); + + Assert.Equal(expected, status.Status); + Assert.Equal(provenance, status.Provenance); + } + + [Fact] + public void Constructor_ThrowsForUnknownStatus() + { + var provenance = new AdvisoryProvenance("test", "status", "value", DateTimeOffset.UtcNow); + Assert.Throws(() => new AffectedPackageStatus("unsupported", provenance)); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs b/src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs new file mode 100644 index 00000000..d313f30c --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs @@ -0,0 +1,52 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class AliasSchemeRegistryTests +{ + [Theory] + [InlineData("cve-2024-1234", AliasSchemes.Cve, "CVE-2024-1234")] + [InlineData("GHSA-xxxx-yyyy-zzzz", AliasSchemes.Ghsa, "GHSA-xxxx-yyyy-zzzz")] + [InlineData("osv-2023-15", AliasSchemes.OsV, "OSV-2023-15")] + [InlineData("jvndb-2023-123456", AliasSchemes.Jvndb, "JVNDB-2023-123456")] + [InlineData("vu#123456", AliasSchemes.Vu, "VU#123456")] + [InlineData("pkg:maven/org.example/app@1.0.0", AliasSchemes.Purl, "pkg:maven/org.example/app@1.0.0")] + [InlineData("cpe:/a:vendor:product:1.0", AliasSchemes.Cpe, "cpe:/a:vendor:product:1.0")] + public void TryNormalize_DetectsSchemeAndCanonicalizes(string input, string expectedScheme, string expectedAlias) + { + var success = AliasSchemeRegistry.TryNormalize(input, out var normalized, out var scheme); + + Assert.True(success); + Assert.Equal(expectedScheme, scheme); + Assert.Equal(expectedAlias, normalized); + } + + [Fact] + public void TryNormalize_ReturnsFalseForUnknownAlias() + { + var success = AliasSchemeRegistry.TryNormalize("custom-identifier", out var normalized, out var scheme); + + Assert.False(success); + Assert.Equal("custom-identifier", normalized); + Assert.Equal(string.Empty, scheme); + } + + [Fact] + public void Validation_NormalizesAliasWhenRecognized() + { + var result = Validation.TryNormalizeAlias(" rhsa-2024:0252 ", out var normalized); + + Assert.True(result); + Assert.NotNull(normalized); + Assert.Equal("RHSA-2024:0252", normalized); + } + + [Fact] + public void Validation_RejectsEmptyAlias() + { + var result = Validation.TryNormalizeAlias(" ", out var normalized); + + Assert.False(result); + Assert.Null(normalized); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs b/src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs new file mode 100644 index 00000000..c52bcbf5 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs @@ -0,0 +1,195 @@ +using System.Collections.Generic; +using System.Globalization; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +internal static class CanonicalExampleFactory +{ + public static IEnumerable<(string Name, Advisory Advisory)> GetExamples() + { + yield return ("nvd-basic", CreateNvdExample()); + yield return ("psirt-overlay", CreatePsirtOverlay()); + yield return ("ghsa-semver", CreateGhsaSemVer()); + yield return ("kev-flag", CreateKevFlag()); + } + + private static Advisory CreateNvdExample() + { + var provenance = Provenance("nvd", "map", "cve-2024-1234", "2024-08-01T12:00:00Z"); + return new Advisory( + advisoryKey: "CVE-2024-1234", + title: "Integer overflow in ExampleCMS", + summary: "An integer overflow in ExampleCMS allows remote attackers to escalate privileges.", + language: "en", + published: ParseDate("2024-07-15T00:00:00Z"), + modified: ParseDate("2024-07-16T10:35:00Z"), + severity: "high", + exploitKnown: false, + aliases: new[] { "CVE-2024-1234" }, + references: new[] + { + new AdvisoryReference( + "https://nvd.nist.gov/vuln/detail/CVE-2024-1234", + kind: "advisory", + sourceTag: "nvd", + summary: "NVD entry", + provenance: provenance), + new AdvisoryReference( + "https://example.org/security/CVE-2024-1234", + kind: "advisory", + sourceTag: "vendor", + summary: "Vendor bulletin", + provenance: Provenance("example", "fetch", "bulletin", "2024-07-14T15:00:00Z")), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: "cpe:/a:examplecms:examplecms:1.0", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange("version", "1.0", "1.0.5", null, null, provenance), + }, + statuses: new[] + { + new AffectedPackageStatus("affected", provenance), + }, + provenance: new[] { provenance }), + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", 9.8, "critical", provenance), + }, + provenance: new[] { provenance }); + } + + private static Advisory CreatePsirtOverlay() + { + var rhsaProv = Provenance("redhat", "map", "rhsa-2024:0252", "2024-05-11T09:00:00Z"); + var cveProv = Provenance("redhat", "enrich", "cve-2024-5678", "2024-05-11T09:05:00Z"); + return new Advisory( + advisoryKey: "RHSA-2024:0252", + title: "Important: kernel security update", + summary: "Updates the Red Hat Enterprise Linux kernel to address CVE-2024-5678.", + language: "en", + published: ParseDate("2024-05-10T19:28:00Z"), + modified: ParseDate("2024-05-11T08:15:00Z"), + severity: "critical", + exploitKnown: false, + aliases: new[] { "RHSA-2024:0252", "CVE-2024-5678" }, + references: new[] + { + new AdvisoryReference( + "https://access.redhat.com/errata/RHSA-2024:0252", + kind: "advisory", + sourceTag: "redhat", + summary: "Red Hat security advisory", + provenance: rhsaProv), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.Rpm, + identifier: "kernel-0:4.18.0-553.el8.x86_64", + platform: "rhel-8", + versionRanges: new[] + { + new AffectedVersionRange("nevra", "0:4.18.0-553.el8", null, null, null, rhsaProv), + }, + statuses: new[] + { + new AffectedPackageStatus("fixed", rhsaProv), + }, + provenance: new[] { rhsaProv, cveProv }), + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:L/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H", 6.7, "medium", rhsaProv), + }, + provenance: new[] { rhsaProv, cveProv }); + } + + private static Advisory CreateGhsaSemVer() + { + var provenance = Provenance("ghsa", "map", "ghsa-aaaa-bbbb-cccc", "2024-03-05T10:00:00Z"); + return new Advisory( + advisoryKey: "GHSA-aaaa-bbbb-cccc", + title: "Prototype pollution in widget.js", + summary: "A crafted payload can pollute Object.prototype leading to RCE.", + language: "en", + published: ParseDate("2024-03-04T00:00:00Z"), + modified: ParseDate("2024-03-04T12:00:00Z"), + severity: "high", + exploitKnown: false, + aliases: new[] { "GHSA-aaaa-bbbb-cccc", "CVE-2024-2222" }, + references: new[] + { + new AdvisoryReference( + "https://github.com/example/widget/security/advisories/GHSA-aaaa-bbbb-cccc", + kind: "advisory", + sourceTag: "ghsa", + summary: "GitHub Security Advisory", + provenance: provenance), + new AdvisoryReference( + "https://github.com/example/widget/commit/abcd1234", + kind: "patch", + sourceTag: "ghsa", + summary: "Patch commit", + provenance: provenance), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/example-widget", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange("semver", null, "2.5.1", null, ">=0.0.0 <2.5.1", provenance), + new AffectedVersionRange("semver", "3.0.0", "3.2.4", null, null, provenance), + }, + statuses: Array.Empty(), + provenance: new[] { provenance }), + }, + cvssMetrics: new[] + { + new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", 8.8, "high", provenance), + }, + provenance: new[] { provenance }); + } + + private static Advisory CreateKevFlag() + { + var provenance = Provenance("cisa-kev", "annotate", "kev", "2024-02-10T09:30:00Z"); + return new Advisory( + advisoryKey: "CVE-2023-9999", + title: "Remote code execution in LegacyServer", + summary: "Unauthenticated RCE due to unsafe deserialization.", + language: "en", + published: ParseDate("2023-11-20T00:00:00Z"), + modified: ParseDate("2024-02-09T16:22:00Z"), + severity: "critical", + exploitKnown: true, + aliases: new[] { "CVE-2023-9999" }, + references: new[] + { + new AdvisoryReference( + "https://www.cisa.gov/known-exploited-vulnerabilities-catalog", + kind: "kev", + sourceTag: "cisa", + summary: "CISA KEV entry", + provenance: provenance), + }, + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static AdvisoryProvenance Provenance(string source, string kind, string value, string recordedAt) + => new(source, kind, value, ParseDate(recordedAt)); + + private static DateTimeOffset ParseDate(string value) + => DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToUniversalTime(); +} diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs b/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs new file mode 100644 index 00000000..d8ff25bf --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs @@ -0,0 +1,57 @@ +using System.Text; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class CanonicalExamplesTests +{ + private static readonly string FixtureRoot = Path.Combine(GetProjectRoot(), "Fixtures"); + private const string UpdateEnvVar = "UPDATE_GOLDENS"; + + [Trait("Category", "GoldenSnapshots")] + [Fact] + public void CanonicalExamplesMatchGoldenSnapshots() + { + Directory.CreateDirectory(FixtureRoot); + var updateGoldens = string.Equals(Environment.GetEnvironmentVariable(UpdateEnvVar), "1", StringComparison.OrdinalIgnoreCase); + var failures = new List(); + + foreach (var (name, advisory) in CanonicalExampleFactory.GetExamples()) + { + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n"); + var fixturePath = Path.Combine(FixtureRoot, $"{name}.json"); + + if (updateGoldens) + { + File.WriteAllText(fixturePath, snapshot); + continue; + } + + if (!File.Exists(fixturePath)) + { + failures.Add($"Missing golden fixture: {fixturePath}. Set {UpdateEnvVar}=1 to generate."); + continue; + } + + var expected = File.ReadAllText(fixturePath).Replace("\r\n", "\n"); + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + failures.Add($"Fixture mismatch for {name}. Set {UpdateEnvVar}=1 to regenerate."); + } + } + + if (failures.Count > 0) + { + var builder = new StringBuilder(); + foreach (var failure in failures) + { + builder.AppendLine(failure); + } + + Assert.Fail(builder.ToString()); + } + } + + private static string GetProjectRoot() + => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..")); +} diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs b/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs new file mode 100644 index 00000000..368c6d5a --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs @@ -0,0 +1,65 @@ +using System; +using System.Linq; +using System.Text.Json; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class CanonicalJsonSerializerTests +{ + [Fact] + public void SerializesPropertiesInDeterministicOrder() + { + var advisory = new Advisory( + advisoryKey: "TEST-321", + title: "Ordering", + summary: null, + language: null, + published: null, + modified: null, + severity: null, + exploitKnown: false, + aliases: new[] { "b", "a" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var json = CanonicalJsonSerializer.Serialize(advisory); + using var document = JsonDocument.Parse(json); + var propertyNames = document.RootElement.EnumerateObject().Select(p => p.Name).ToArray(); + + var sorted = propertyNames.OrderBy(name => name, StringComparer.Ordinal).ToArray(); + Assert.Equal(sorted, propertyNames); + } + + [Fact] + public void SnapshotSerializerProducesStableOutput() + { + var advisory = new Advisory( + advisoryKey: "TEST-999", + title: "Snapshot", + summary: "Example", + language: "EN", + published: DateTimeOffset.Parse("2024-06-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-06-01T01:00:00Z"), + severity: "high", + exploitKnown: false, + aliases: new[] { "ALIAS-1" }, + references: new[] + { + new AdvisoryReference("https://example.com/a", "advisory", null, null, AdvisoryProvenance.Empty), + }, + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var snap1 = SnapshotSerializer.ToSnapshot(advisory); + var snap2 = SnapshotSerializer.ToSnapshot(advisory); + + Assert.Equal(snap1, snap2); + var normalized1 = snap1.Replace("\r\n", "\n"); + var normalized2 = snap2.Replace("\r\n", "\n"); + Assert.Equal(normalized1, normalized2); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs b/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs new file mode 100644 index 00000000..7b9099e4 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs @@ -0,0 +1,28 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Models.Tests; + +public sealed class SeverityNormalizationTests +{ + [Theory] + [InlineData("CRITICAL", "critical")] + [InlineData("Important", "high")] + [InlineData("moderate", "medium")] + [InlineData("Minor", "low")] + [InlineData("Info", "informational")] + [InlineData("negligible", "none")] + [InlineData("unknown", "unknown")] + [InlineData("custom-level", "custom-level")] + public void Normalize_ReturnsExpectedCanonicalValue(string input, string expected) + { + var normalized = SeverityNormalization.Normalize(input); + Assert.Equal(expected, normalized); + } + + [Fact] + public void Normalize_ReturnsNullWhenInputNullOrWhitespace() + { + Assert.Null(SeverityNormalization.Normalize(null)); + Assert.Null(SeverityNormalization.Normalize(" ")); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj b/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj new file mode 100644 index 00000000..6d0b9223 --- /dev/null +++ b/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + + + + + diff --git a/src/StellaOps.Feedser.Models/AGENTS.md b/src/StellaOps.Feedser.Models/AGENTS.md new file mode 100644 index 00000000..cee8c137 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AGENTS.md @@ -0,0 +1,30 @@ +# AGENTS +## Role +Canonical data model for normalized advisories and all downstream serialization. Source of truth for merge/export. +## Scope +- Canonical types: Advisory, AdvisoryReference, CvssMetric, AffectedPackage, AffectedVersionRange, AdvisoryProvenance. +- Invariants: stable ordering, culture-invariant serialization, UTC timestamps, deterministic equality semantics. +- Field semantics: preserve all aliases/references; ranges per ecosystem (NEVRA/EVR/SemVer); provenance on every mapped field. +- Backward/forward compatibility: additive evolution; versioned DTOs where needed; no breaking field renames. +- Detailed field coverage documented in `CANONICAL_RECORDS.md`; update alongside model changes. +## Participants +- Source connectors map external DTOs into these types. +- Merge engine composes/overrides AffectedPackage sets and consolidates references/aliases. +- Exporters serialize canonical documents deterministically. +## Interfaces & contracts +- Null-object statics: Advisory.Empty, AdvisoryReference.Empty, CvssMetric.Empty. +- AffectedPackage.Type describes semantics (e.g., rpm, deb, cpe, semver). Identifier is stable (e.g., NEVRA, PURL, CPE). +- Version ranges list is ordered by introduction then fix; provenance identifies source/kind/value/recordedAt. +- Alias schemes must include CVE, GHSA, OSV, JVN/JVNDB, BDU, VU(CERT/CC), MSRC, CISCO-SA, ORACLE-CPU, APSB/APA, APPLE-HT, CHROMIUM-POST, VMSA, RHSA, USN, DSA, SUSE-SU, ICSA, CWE, CPE, PURL. +## In/Out of scope +In: data shapes, invariants, helpers for canonical serialization and comparison. +Out: fetching/parsing external schemas, storage, HTTP. +## Observability & security expectations +- No secrets; purely in-memory types. +- Provide debug renders for test snapshots (canonical JSON). +- Emit model version identifiers in logs when canonical structures change; keep adapters for older readers until deprecated. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Models.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Models/Advisory.cs b/src/StellaOps.Feedser.Models/Advisory.cs new file mode 100644 index 00000000..e1e393bd --- /dev/null +++ b/src/StellaOps.Feedser.Models/Advisory.cs @@ -0,0 +1,145 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonical advisory document produced after merge. Collections are pre-sorted for deterministic serialization. +/// +public sealed record Advisory +{ + public static Advisory Empty { get; } = new( + advisoryKey: "unknown", + title: "", + summary: null, + language: null, + published: null, + modified: null, + severity: null, + exploitKnown: false, + aliases: Array.Empty(), + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + public Advisory( + string advisoryKey, + string title, + string? summary, + string? language, + DateTimeOffset? published, + DateTimeOffset? modified, + string? severity, + bool exploitKnown, + IEnumerable? aliases, + IEnumerable? references, + IEnumerable? affectedPackages, + IEnumerable? cvssMetrics, + IEnumerable? provenance) + { + AdvisoryKey = Validation.EnsureNotNullOrWhiteSpace(advisoryKey, nameof(advisoryKey)); + Title = Validation.EnsureNotNullOrWhiteSpace(title, nameof(title)); + Summary = Validation.TrimToNull(summary); + Language = Validation.TrimToNull(language)?.ToLowerInvariant(); + Published = published?.ToUniversalTime(); + Modified = modified?.ToUniversalTime(); + Severity = SeverityNormalization.Normalize(severity); + ExploitKnown = exploitKnown; + + Aliases = (aliases ?? Array.Empty()) + .Select(static alias => Validation.TryNormalizeAlias(alias, out var normalized) ? normalized! : null) + .Where(static alias => alias is not null) + .Distinct(StringComparer.Ordinal) + .OrderBy(static alias => alias, StringComparer.Ordinal) + .Select(static alias => alias!) + .ToImmutableArray(); + + References = (references ?? Array.Empty()) + .Where(static reference => reference is not null) + .OrderBy(static reference => reference.Url, StringComparer.Ordinal) + .ThenBy(static reference => reference.Kind, StringComparer.Ordinal) + .ThenBy(static reference => reference.SourceTag, StringComparer.Ordinal) + .ThenBy(static reference => reference.Provenance.RecordedAt) + .ToImmutableArray(); + + AffectedPackages = (affectedPackages ?? Array.Empty()) + .Where(static package => package is not null) + .OrderBy(static package => package.Type, StringComparer.Ordinal) + .ThenBy(static package => package.Identifier, StringComparer.Ordinal) + .ThenBy(static package => package.Platform, StringComparer.Ordinal) + .ToImmutableArray(); + + CvssMetrics = (cvssMetrics ?? Array.Empty()) + .Where(static metric => metric is not null) + .OrderBy(static metric => metric.Version, StringComparer.Ordinal) + .ThenBy(static metric => metric.Vector, StringComparer.Ordinal) + .ToImmutableArray(); + + Provenance = (provenance ?? Array.Empty()) + .Where(static p => p is not null) + .OrderBy(static p => p.Source, StringComparer.Ordinal) + .ThenBy(static p => p.Kind, StringComparer.Ordinal) + .ThenBy(static p => p.RecordedAt) + .ToImmutableArray(); + } + + [JsonConstructor] + public Advisory( + string advisoryKey, + string title, + string? summary, + string? language, + DateTimeOffset? published, + DateTimeOffset? modified, + string? severity, + bool exploitKnown, + ImmutableArray aliases, + ImmutableArray references, + ImmutableArray affectedPackages, + ImmutableArray cvssMetrics, + ImmutableArray provenance) + : this( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown, + aliases.IsDefault ? null : aliases.AsEnumerable(), + references.IsDefault ? null : references.AsEnumerable(), + affectedPackages.IsDefault ? null : affectedPackages.AsEnumerable(), + cvssMetrics.IsDefault ? null : cvssMetrics.AsEnumerable(), + provenance.IsDefault ? null : provenance.AsEnumerable()) + { + } + + public string AdvisoryKey { get; } + + public string Title { get; } + + public string? Summary { get; } + + public string? Language { get; } + + public DateTimeOffset? Published { get; } + + public DateTimeOffset? Modified { get; } + + public string? Severity { get; } + + public bool ExploitKnown { get; } + + public ImmutableArray Aliases { get; } + + public ImmutableArray References { get; } + + public ImmutableArray AffectedPackages { get; } + + public ImmutableArray CvssMetrics { get; } + + public ImmutableArray Provenance { get; } +} diff --git a/src/StellaOps.Feedser.Models/AdvisoryProvenance.cs b/src/StellaOps.Feedser.Models/AdvisoryProvenance.cs new file mode 100644 index 00000000..f789a759 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AdvisoryProvenance.cs @@ -0,0 +1,28 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Describes the origin of a canonical field and how/when it was captured. +/// +public sealed record AdvisoryProvenance +{ + public static AdvisoryProvenance Empty { get; } = new("unknown", "unspecified", string.Empty, DateTimeOffset.UnixEpoch); + + [JsonConstructor] + public AdvisoryProvenance(string source, string kind, string value, DateTimeOffset recordedAt) + { + Source = Validation.EnsureNotNullOrWhiteSpace(source, nameof(source)); + Kind = Validation.EnsureNotNullOrWhiteSpace(kind, nameof(kind)); + Value = Validation.TrimToNull(value); + RecordedAt = recordedAt.ToUniversalTime(); + } + + public string Source { get; } + + public string Kind { get; } + + public string? Value { get; } + + public DateTimeOffset RecordedAt { get; } +} diff --git a/src/StellaOps.Feedser.Models/AdvisoryReference.cs b/src/StellaOps.Feedser.Models/AdvisoryReference.cs new file mode 100644 index 00000000..c5f18650 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AdvisoryReference.cs @@ -0,0 +1,36 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonical external reference associated with an advisory. +/// +public sealed record AdvisoryReference +{ + public static AdvisoryReference Empty { get; } = new("https://invalid.local/", kind: null, sourceTag: null, summary: null, provenance: AdvisoryProvenance.Empty); + + [JsonConstructor] + public AdvisoryReference(string url, string? kind, string? sourceTag, string? summary, AdvisoryProvenance provenance) + { + if (!Validation.LooksLikeHttpUrl(url)) + { + throw new ArgumentException("Reference URL must be an absolute http(s) URI.", nameof(url)); + } + + Url = url; + Kind = Validation.TrimToNull(kind); + SourceTag = Validation.TrimToNull(sourceTag); + Summary = Validation.TrimToNull(summary); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + public string Url { get; } + + public string? Kind { get; } + + public string? SourceTag { get; } + + public string? Summary { get; } + + public AdvisoryProvenance Provenance { get; } +} diff --git a/src/StellaOps.Feedser.Models/AffectedPackage.cs b/src/StellaOps.Feedser.Models/AffectedPackage.cs new file mode 100644 index 00000000..bc26c0ac --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedPackage.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonical affected package descriptor with deterministic ordering of ranges and provenance. +/// +public sealed record AffectedPackage +{ + public static AffectedPackage Empty { get; } = new( + AffectedPackageTypes.SemVer, + identifier: "unknown", + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: Array.Empty()); + + [JsonConstructor] + public AffectedPackage( + string type, + string identifier, + string? platform = null, + IEnumerable? versionRanges = null, + IEnumerable? statuses = null, + IEnumerable? provenance = null) + { + Type = Validation.EnsureNotNullOrWhiteSpace(type, nameof(type)).ToLowerInvariant(); + Identifier = Validation.EnsureNotNullOrWhiteSpace(identifier, nameof(identifier)); + Platform = Validation.TrimToNull(platform); + + VersionRanges = (versionRanges ?? Array.Empty()) + .Distinct(AffectedVersionRangeEqualityComparer.Instance) + .OrderBy(static range => range, AffectedVersionRangeComparer.Instance) + .ToImmutableArray(); + + Statuses = (statuses ?? Array.Empty()) + .Where(static status => status is not null) + .Distinct(AffectedPackageStatusEqualityComparer.Instance) + .OrderBy(static status => status.Status, StringComparer.Ordinal) + .ThenBy(static status => status.Provenance.Source, StringComparer.Ordinal) + .ThenBy(static status => status.Provenance.Kind, StringComparer.Ordinal) + .ThenBy(static status => status.Provenance.RecordedAt) + .ToImmutableArray(); + + Provenance = (provenance ?? Array.Empty()) + .Where(static p => p is not null) + .OrderBy(static p => p.Source, StringComparer.Ordinal) + .ThenBy(static p => p.Kind, StringComparer.Ordinal) + .ThenBy(static p => p.RecordedAt) + .ToImmutableArray(); + } + + /// + /// Semantic type of the coordinates (rpm, deb, cpe, semver, vendor, ics-vendor). + /// + public string Type { get; } + + /// + /// Canonical identifier for the package (NEVRA, PackageURL, CPE string, vendor slug, etc.). + /// + public string Identifier { get; } + + public string? Platform { get; } + + public ImmutableArray VersionRanges { get; } + + public ImmutableArray Statuses { get; } + + public ImmutableArray Provenance { get; } +} + +/// +/// Known values for . +/// +public static class AffectedPackageTypes +{ + public const string Rpm = "rpm"; + public const string Deb = "deb"; + public const string Cpe = "cpe"; + public const string SemVer = "semver"; + public const string Vendor = "vendor"; + public const string IcsVendor = "ics-vendor"; +} diff --git a/src/StellaOps.Feedser.Models/AffectedPackageStatus.cs b/src/StellaOps.Feedser.Models/AffectedPackageStatus.cs new file mode 100644 index 00000000..d8660395 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedPackageStatus.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Represents a vendor-supplied status tag for an affected package when a concrete version range is unavailable or supplementary. +/// +public sealed record AffectedPackageStatus +{ + [JsonConstructor] + public AffectedPackageStatus(string status, AdvisoryProvenance provenance) + { + Status = AffectedPackageStatusCatalog.Normalize(status); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + public string Status { get; } + + public AdvisoryProvenance Provenance { get; } +} + +public sealed class AffectedPackageStatusEqualityComparer : IEqualityComparer +{ + public static AffectedPackageStatusEqualityComparer Instance { get; } = new(); + + public bool Equals(AffectedPackageStatus? x, AffectedPackageStatus? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Status, y.Status, StringComparison.Ordinal) + && EqualityComparer.Default.Equals(x.Provenance, y.Provenance); + } + + public int GetHashCode(AffectedPackageStatus obj) + => HashCode.Combine(obj.Status, obj.Provenance); +} diff --git a/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs b/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs new file mode 100644 index 00000000..a5a232f9 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Models; + +/// +/// Central registry of allowed affected-package status labels to keep connectors consistent. +/// +public static class AffectedPackageStatusCatalog +{ + public const string KnownAffected = "known_affected"; + public const string KnownNotAffected = "known_not_affected"; + public const string UnderInvestigation = "under_investigation"; + public const string Fixed = "fixed"; + public const string FirstFixed = "first_fixed"; + public const string Mitigated = "mitigated"; + public const string NotApplicable = "not_applicable"; + public const string Affected = "affected"; + public const string NotAffected = "not_affected"; + public const string Pending = "pending"; + public const string Unknown = "unknown"; + + private static readonly HashSet AllowedStatuses = new(StringComparer.OrdinalIgnoreCase) + { + KnownAffected, + KnownNotAffected, + UnderInvestigation, + Fixed, + FirstFixed, + Mitigated, + NotApplicable, + Affected, + NotAffected, + Pending, + Unknown, + }; + + public static IReadOnlyCollection Allowed => AllowedStatuses; + + public static string Normalize(string status) + { + if (string.IsNullOrWhiteSpace(status)) + { + throw new ArgumentException("Status must be provided.", nameof(status)); + } + + var token = status.Trim().ToLowerInvariant().Replace(' ', '_').Replace('-', '_'); + if (!AllowedStatuses.Contains(token)) + { + throw new ArgumentOutOfRangeException(nameof(status), status, "Status is not part of the allowed affected-package status glossary."); + } + + return token; + } +} diff --git a/src/StellaOps.Feedser.Models/AffectedVersionRange.cs b/src/StellaOps.Feedser.Models/AffectedVersionRange.cs new file mode 100644 index 00000000..4a211beb --- /dev/null +++ b/src/StellaOps.Feedser.Models/AffectedVersionRange.cs @@ -0,0 +1,145 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Describes a contiguous range of versions impacted by an advisory. +/// +public sealed record AffectedVersionRange +{ + [JsonConstructor] + public AffectedVersionRange( + string rangeKind, + string? introducedVersion, + string? fixedVersion, + string? lastAffectedVersion, + string? rangeExpression, + AdvisoryProvenance provenance) + { + RangeKind = Validation.EnsureNotNullOrWhiteSpace(rangeKind, nameof(rangeKind)).ToLowerInvariant(); + IntroducedVersion = Validation.TrimToNull(introducedVersion); + FixedVersion = Validation.TrimToNull(fixedVersion); + LastAffectedVersion = Validation.TrimToNull(lastAffectedVersion); + RangeExpression = Validation.TrimToNull(rangeExpression); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + /// + /// Semantic kind of the range (e.g., semver, nevra, evr). + /// + public string RangeKind { get; } + + /// + /// Inclusive version where impact begins. + /// + public string? IntroducedVersion { get; } + + /// + /// Exclusive version where impact ends due to a fix. + /// + public string? FixedVersion { get; } + + /// + /// Inclusive upper bound where the vendor reports exposure (when no fix available). + /// + public string? LastAffectedVersion { get; } + + /// + /// Normalized textual representation of the range (fallback). + /// + public string? RangeExpression { get; } + + public AdvisoryProvenance Provenance { get; } + + public string CreateDeterministicKey() + => string.Join('|', RangeKind, IntroducedVersion ?? string.Empty, FixedVersion ?? string.Empty, LastAffectedVersion ?? string.Empty, RangeExpression ?? string.Empty); +} + +/// +/// Deterministic comparer for version ranges. Orders by introduced, fixed, last affected, expression, kind. +/// +public sealed class AffectedVersionRangeComparer : IComparer +{ + public static AffectedVersionRangeComparer Instance { get; } = new(); + + private static readonly StringComparer Comparer = StringComparer.Ordinal; + + public int Compare(AffectedVersionRange? x, AffectedVersionRange? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var compare = Comparer.Compare(x.IntroducedVersion, y.IntroducedVersion); + if (compare != 0) + { + return compare; + } + + compare = Comparer.Compare(x.FixedVersion, y.FixedVersion); + if (compare != 0) + { + return compare; + } + + compare = Comparer.Compare(x.LastAffectedVersion, y.LastAffectedVersion); + if (compare != 0) + { + return compare; + } + + compare = Comparer.Compare(x.RangeExpression, y.RangeExpression); + if (compare != 0) + { + return compare; + } + + return Comparer.Compare(x.RangeKind, y.RangeKind); + } +} + +/// +/// Equality comparer that ignores provenance differences. +/// +public sealed class AffectedVersionRangeEqualityComparer : IEqualityComparer +{ + public static AffectedVersionRangeEqualityComparer Instance { get; } = new(); + + public bool Equals(AffectedVersionRange? x, AffectedVersionRange? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.RangeKind, y.RangeKind, StringComparison.Ordinal) + && string.Equals(x.IntroducedVersion, y.IntroducedVersion, StringComparison.Ordinal) + && string.Equals(x.FixedVersion, y.FixedVersion, StringComparison.Ordinal) + && string.Equals(x.LastAffectedVersion, y.LastAffectedVersion, StringComparison.Ordinal) + && string.Equals(x.RangeExpression, y.RangeExpression, StringComparison.Ordinal); + } + + public int GetHashCode(AffectedVersionRange obj) + => HashCode.Combine( + obj.RangeKind, + obj.IntroducedVersion, + obj.FixedVersion, + obj.LastAffectedVersion, + obj.RangeExpression); +} diff --git a/src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs b/src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs new file mode 100644 index 00000000..52bbe391 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Models; + +public static class AliasSchemeRegistry +{ + private sealed record AliasScheme( + string Name, + Func Predicate, + Func Normalizer); + +private static readonly AliasScheme[] SchemeDefinitions = + { + BuildScheme(AliasSchemes.Cve, alias => alias is not null && Matches(CvERegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "CVE")), + BuildScheme(AliasSchemes.Ghsa, alias => alias is not null && Matches(GhsaRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "GHSA")), + BuildScheme(AliasSchemes.OsV, alias => alias is not null && Matches(OsVRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "OSV")), + BuildScheme(AliasSchemes.Jvn, alias => alias is not null && Matches(JvnRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "JVN")), + BuildScheme(AliasSchemes.Jvndb, alias => alias is not null && Matches(JvndbRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "JVNDB")), + BuildScheme(AliasSchemes.Bdu, alias => alias is not null && Matches(BduRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "BDU")), + BuildScheme(AliasSchemes.Vu, alias => alias is not null && alias.StartsWith("VU#", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "VU", preserveSeparator: '#')), + BuildScheme(AliasSchemes.Msrc, alias => alias is not null && alias.StartsWith("MSRC-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "MSRC")), + BuildScheme(AliasSchemes.CiscoSa, alias => alias is not null && alias.StartsWith("CISCO-SA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "CISCO-SA")), + BuildScheme(AliasSchemes.OracleCpu, alias => alias is not null && alias.StartsWith("ORACLE-CPU", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "ORACLE-CPU")), + BuildScheme(AliasSchemes.Apsb, alias => alias is not null && alias.StartsWith("APSB-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "APSB")), + BuildScheme(AliasSchemes.Apa, alias => alias is not null && alias.StartsWith("APA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "APA")), + BuildScheme(AliasSchemes.AppleHt, alias => alias is not null && alias.StartsWith("APPLE-HT", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "APPLE-HT")), + BuildScheme(AliasSchemes.ChromiumPost, alias => alias is not null && (alias.StartsWith("CHROMIUM-POST", StringComparison.OrdinalIgnoreCase) || alias.StartsWith("CHROMIUM:", StringComparison.OrdinalIgnoreCase)), NormalizeChromium), + BuildScheme(AliasSchemes.Vmsa, alias => alias is not null && alias.StartsWith("VMSA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "VMSA")), + BuildScheme(AliasSchemes.Rhsa, alias => alias is not null && alias.StartsWith("RHSA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "RHSA")), + BuildScheme(AliasSchemes.Usn, alias => alias is not null && alias.StartsWith("USN-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "USN")), + BuildScheme(AliasSchemes.Dsa, alias => alias is not null && alias.StartsWith("DSA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "DSA")), + BuildScheme(AliasSchemes.SuseSu, alias => alias is not null && alias.StartsWith("SUSE-SU-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "SUSE-SU")), + BuildScheme(AliasSchemes.Icsa, alias => alias is not null && alias.StartsWith("ICSA-", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "ICSA")), + BuildScheme(AliasSchemes.Cwe, alias => alias is not null && Matches(CweRegex, alias), alias => alias is null ? string.Empty : NormalizePrefix(alias, "CWE")), + BuildScheme(AliasSchemes.Cpe, alias => alias is not null && alias.StartsWith("cpe:", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "cpe", uppercase:false)), + BuildScheme(AliasSchemes.Purl, alias => alias is not null && alias.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase), alias => NormalizePrefix(alias, "pkg", uppercase:false)), + }; + + private static AliasScheme BuildScheme(string name, Func predicate, Func normalizer) + => new( + name, + predicate, + alias => normalizer(alias)); + + private static readonly ImmutableHashSet SchemeNames = SchemeDefinitions + .Select(static scheme => scheme.Name) + .ToImmutableHashSet(StringComparer.OrdinalIgnoreCase); + + private static readonly Regex CvERegex = new("^CVE-\\d{4}-\\d{4,}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex GhsaRegex = new("^GHSA-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex OsVRegex = new("^OSV-\\d{4}-\\d+$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex JvnRegex = new("^JVN-\\d{4}-\\d{6}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex JvndbRegex = new("^JVNDB-\\d{4}-\\d{6}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex BduRegex = new("^BDU-\\d{4}-\\d+$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private static readonly Regex CweRegex = new("^CWE-\\d+$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + + public static IReadOnlyCollection KnownSchemes => SchemeNames; + + public static bool IsKnownScheme(string? scheme) + => !string.IsNullOrWhiteSpace(scheme) && SchemeNames.Contains(scheme); + + public static bool TryGetScheme(string? alias, out string scheme) + { + if (string.IsNullOrWhiteSpace(alias)) + { + scheme = string.Empty; + return false; + } + + var candidate = alias.Trim(); + foreach (var entry in SchemeDefinitions) + { + if (entry.Predicate(candidate)) + { + scheme = entry.Name; + return true; + } + } + + scheme = string.Empty; + return false; + } + + public static bool TryNormalize(string? alias, out string normalized, out string scheme) + { + normalized = string.Empty; + scheme = string.Empty; + + if (string.IsNullOrWhiteSpace(alias)) + { + return false; + } + + var candidate = alias.Trim(); + foreach (var entry in SchemeDefinitions) + { + if (entry.Predicate(candidate)) + { + scheme = entry.Name; + normalized = entry.Normalizer(candidate); + return true; + } + } + + normalized = candidate; + return false; + } + + private static string NormalizePrefix(string? alias, string prefix, bool uppercase = true, char? preserveSeparator = null) + { + if (string.IsNullOrWhiteSpace(alias)) + { + return string.Empty; + } + + var comparison = StringComparison.OrdinalIgnoreCase; + if (!alias.StartsWith(prefix, comparison)) + { + return uppercase ? alias : alias.ToLowerInvariant(); + } + + var remainder = alias[prefix.Length..]; + if (preserveSeparator is { } separator && remainder.Length > 0 && remainder[0] != separator) + { + // Edge case: alias is expected to use a specific separator but does not – return unchanged. + return uppercase ? prefix.ToUpperInvariant() + remainder : prefix + remainder; + } + + var normalizedPrefix = uppercase ? prefix.ToUpperInvariant() : prefix.ToLowerInvariant(); + return normalizedPrefix + remainder; + } + + private static string NormalizeChromium(string? alias) + { + if (string.IsNullOrWhiteSpace(alias)) + { + return string.Empty; + } + + if (alias.StartsWith("CHROMIUM-POST", StringComparison.OrdinalIgnoreCase)) + { + return NormalizePrefix(alias, "CHROMIUM-POST"); + } + + if (alias.StartsWith("CHROMIUM:", StringComparison.OrdinalIgnoreCase)) + { + var remainder = alias["CHROMIUM".Length..]; + return "CHROMIUM" + remainder; + } + + return alias; + } + private static bool Matches(Regex? regex, string? candidate) + { + if (regex is null || string.IsNullOrWhiteSpace(candidate)) + { + return false; + } + + return regex.IsMatch(candidate); + } +} diff --git a/src/StellaOps.Feedser.Models/AliasSchemes.cs b/src/StellaOps.Feedser.Models/AliasSchemes.cs new file mode 100644 index 00000000..3541caa0 --- /dev/null +++ b/src/StellaOps.Feedser.Models/AliasSchemes.cs @@ -0,0 +1,31 @@ +namespace StellaOps.Feedser.Models; + +/// +/// Well-known alias scheme identifiers referenced throughout the pipeline. +/// +public static class AliasSchemes +{ + public const string Cve = "CVE"; + public const string Ghsa = "GHSA"; + public const string OsV = "OSV"; + public const string Jvn = "JVN"; + public const string Jvndb = "JVNDB"; + public const string Bdu = "BDU"; + public const string Vu = "VU"; + public const string Msrc = "MSRC"; + public const string CiscoSa = "CISCO-SA"; + public const string OracleCpu = "ORACLE-CPU"; + public const string Apsb = "APSB"; + public const string Apa = "APA"; + public const string AppleHt = "APPLE-HT"; + public const string ChromiumPost = "CHROMIUM-POST"; + public const string Vmsa = "VMSA"; + public const string Rhsa = "RHSA"; + public const string Usn = "USN"; + public const string Dsa = "DSA"; + public const string SuseSu = "SUSE-SU"; + public const string Icsa = "ICSA"; + public const string Cwe = "CWE"; + public const string Cpe = "CPE"; + public const string Purl = "PURL"; +} diff --git a/src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md b/src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md new file mode 100644 index 00000000..4cb589a8 --- /dev/null +++ b/src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md @@ -0,0 +1,41 @@ +# Canonical Model Backward-Compatibility Playbook + +This playbook captures the policies and workflow required when evolving the canonical +`StellaOps.Feedser.Models` surface. + +## Principles + +- **Additive by default** – breaking field removals/renames are not allowed without a staged + migration plan. +- **Version-the-writer** – any change to serialization that affects downstream consumers must bump + the exporter version string and update `CANONICAL_RECORDS.md`. +- **Schema-first** – update documentation (`CANONICAL_RECORDS.md`) and corresponding tests before + shipping new fields. +- **Dual-read period** – when introducing a new field, keep old readers working by: + 1. Making the field optional in the canonical model. + 2. Providing default behavior in exporters/mergers when the field is absent. + 3. Communicating via release notes and toggles when the field will become required. + +## Workflow for Changes + +1. **Proposal** – raise an issue describing the motivation, affected records, and compatibility + impact. Link to the relevant task in `TASKS.md`. +2. **Docs + Tests first** – update `CANONICAL_RECORDS.md`, add/adjust golden fixtures, and extend + regression tests (hash comparisons, snapshot assertions) to capture the new shape. +3. **Implementation** – introduce the model change along with migration logic (e.g., mergers filling + defaults, exporters emitting the new payload). +4. **Exporter bump** – update exporter version manifests (`ExporterVersion.GetVersion`) whenever the + serialized payload differs. +5. **Announcement** – document the change in release notes, highlighting optional vs. required + timelines. +6. **Cleanup** – once consumers have migrated, remove transitional logic and update docs/tests to + reflect the permanent shape. + +## Testing Checklist + +- `StellaOps.Feedser.Models.Tests` – update unit tests and golden examples. +- `Serialization determinism` – ensure the hash regression tests cover the new fields. +- Exporter integration (`Json`, `TrivyDb`) – confirm manifests include provenance + tree metadata + for the new shape. + +Following this playbook keeps canonical payloads stable while allowing incremental evolution. diff --git a/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md b/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md new file mode 100644 index 00000000..2b9be57e --- /dev/null +++ b/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md @@ -0,0 +1,128 @@ +# Canonical Record Definitions + +> Source of truth for the normalized advisory schema emitted by `StellaOps.Feedser.Models`. +> Keep this document in sync with the public record types under `StellaOps.Feedser.Models` and +> update it whenever a new field is introduced or semantics change. + +## Advisory + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `advisoryKey` | string | yes | Globally unique identifier selected by the merge layer (often a CVE/GHSA/vendor key). Stored lowercased unless vendor casing is significant. | +| `title` | string | yes | Human readable title. Must be non-empty and trimmed. | +| `summary` | string? | optional | Short description; trimmed to `null` when empty. | +| `language` | string? | optional | ISO language code (lowercase). | +| `published` | DateTimeOffset? | optional | UTC timestamp when vendor originally published. | +| `modified` | DateTimeOffset? | optional | UTC timestamp when vendor last updated. | +| `severity` | string? | optional | Normalized severity label (`critical`, `high`, etc.). | +| `exploitKnown` | bool | yes | Whether KEV/other sources confirm active exploitation. | +| `aliases` | string[] | yes | Sorted, de-duplicated list of normalized aliases (see [Alias Schemes](#alias-schemes)). | +| `references` | AdvisoryReference[] | yes | Deterministically ordered reference set. | +| `affectedPackages` | AffectedPackage[] | yes | Deterministically ordered affected packages. | +| `cvssMetrics` | CvssMetric[] | yes | Deterministically ordered CVSS metrics (v3, v4 first). | +| `provenance` | AdvisoryProvenance[] | yes | Normalized provenance entries sorted by source then kind then recorded timestamp. | + +### Invariants +- Collections are immutable (`ImmutableArray`) and always sorted deterministically. +- `AdvisoryKey` and `Title` are mandatory and trimmed. +- All timestamps are stored as UTC. +- Aliases and references leverage helper registries for validation. + +## AdvisoryReference + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `url` | string | yes | Absolute HTTP/HTTPS URL. | +| `kind` | string? | optional | Categorized reference role (e.g. `advisory`, `patch`, `changelog`). | +| `sourceTag` | string? | optional | Free-form tag identifying originating source. | +| `summary` | string? | optional | Short description. | +| `provenance` | AdvisoryProvenance | yes | Provenance entry describing how the reference was mapped. | + +Deterministic ordering: by `url`, then `kind`, then `sourceTag`, then `provenance.RecordedAt`. + +## AffectedPackage + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `type` | string | yes | Semantic type (`semver`, `rpm`, `deb`, `purl`, `cpe`, etc.). Lowercase. | +| `identifier` | string | yes | Canonical identifier (package name, PURL, CPE, NEVRA, etc.). | +| `platform` | string? | optional | Explicit platform / distro (e.g. `ubuntu`, `rhel-8`). | +| `versionRanges` | AffectedVersionRange[] | yes | Deduplicated + sorted by introduced/fixed/last/expr/kind. | +| `statuses` | AffectedPackageStatus[] | yes | Optional status flags (e.g. `fixed`, `affected`). | +| `provenance` | AdvisoryProvenance[] | yes | Provenance entries for package level metadata. | + +Deterministic ordering: packages sorted by `type`, then `identifier`, then `platform` (ordinal). + +## AffectedVersionRange + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `rangeKind` | string | yes | Classification of range semantics (`semver`, `evr`, `nevra`, `version`, `purl`). Lowercase. | +| `introducedVersion` | string? | optional | Inclusive lower bound when impact begins. | +| `fixedVersion` | string? | optional | Exclusive bounding version containing the fix. | +| `lastAffectedVersion` | string? | optional | Inclusive upper bound when no fix exists. | +| `rangeExpression` | string? | optional | Normalized textual expression for non-simple ranges. | +| `provenance` | AdvisoryProvenance | yes | Provenance entry for the range. | + +Comparers/equality ignore provenance differences. + +## CvssMetric + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `version` | string | yes | `2.0`, `3.0`, `3.1`, `4.0`, etc. | +| `vector` | string | yes | Official CVSS vector string. | +| `score` | double | yes | CVSS base score (0.0-10.0). | +| `severity` | string | yes | Severity label mapped from score or vendor metadata. | +| `provenance` | AdvisoryProvenance | yes | Provenance entry. | + +Sorted by version then vector for determinism. + +## AdvisoryProvenance + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `source` | string | yes | Logical source identifier (`nvd`, `redhat`, `osv`, etc.). | +| `kind` | string | yes | Operation performed (`fetch`, `parse`, `map`, `merge`, `enrich`). | +| `detail` | string | optional | Free-form pipeline detail (parser identifier, rule set). | +| `recordedAt` | DateTimeOffset | yes | UTC timestamp when provenance was captured. | + +### Provenance Mask Expectations +Each canonical field is expected to carry at least one provenance entry derived from the +responsible pipeline stage. When aggregating provenance from subcomponents (e.g., affected package +ranges), merge code should ensure: + +- Advisory level provenance documents the source document and merge actions. +- References, packages, ranges, and metrics each include their own provenance entry reflecting + the most specific source (vendor feed, computed normalization, etc.). +- Export-specific metadata (digest manifests, offline bundles) include exporter version alongside + the builder metadata. + +## Alias Schemes + +Supported alias scheme prefixes: + +- `CVE-` +- `GHSA-` +- `OSV-` +- `JVN-`, `JVNDB-` +- `BDU-` +- `VU#` +- `MSRC-` +- `CISCO-SA-` +- `ORACLE-CPU` +- `APSB-`, `APA-` +- `APPLE-HT` +- `CHROMIUM:` / `CHROMIUM-` +- `VMSA-` +- `RHSA-` +- `USN-` +- `DSA-` +- `SUSE-SU-` +- `ICSA-` +- `CWE-` +- `cpe:` +- `pkg:` (Package URL / PURL) + +The registry exposed via `AliasSchemes` and `AliasSchemeRegistry` can be used to validate aliases and +drive downstream conditionals without re-implementing pattern rules. diff --git a/src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs b/src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs new file mode 100644 index 00000000..2c671406 --- /dev/null +++ b/src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs @@ -0,0 +1,91 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace StellaOps.Feedser.Models; + +/// +/// Deterministic JSON serializer tuned for canonical advisory output. +/// +public static class CanonicalJsonSerializer +{ + private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); + private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); + + public static string Serialize(T value) + => JsonSerializer.Serialize(value, CompactOptions); + + public static string SerializeIndented(T value) + => JsonSerializer.Serialize(value, PrettyOptions); + + public static Advisory Normalize(Advisory advisory) + => new( + advisory.AdvisoryKey, + advisory.Title, + advisory.Summary, + advisory.Language, + advisory.Published, + advisory.Modified, + advisory.Severity, + advisory.ExploitKnown, + advisory.Aliases, + advisory.References, + advisory.AffectedPackages, + advisory.CvssMetrics, + advisory.Provenance); + + public static T Deserialize(string json) + => JsonSerializer.Deserialize(json, PrettyOptions)! + ?? throw new InvalidOperationException($"Unable to deserialize type {typeof(T).Name}."); + + private static JsonSerializerOptions CreateOptions(bool writeIndented) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + WriteIndented = writeIndented, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); + options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false)); + return options; + } + + private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver + { + private readonly IJsonTypeInfoResolver _inner; + + public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) + { + var info = _inner.GetTypeInfo(type, options); + if (info is null) + { + throw new InvalidOperationException($"Unable to resolve JsonTypeInfo for '{type}'."); + } + if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) + { + var ordered = info.Properties + .OrderBy(static property => property.Name, StringComparer.Ordinal) + .ToArray(); + + info.Properties.Clear(); + foreach (var property in ordered) + { + info.Properties.Add(property); + } + } + + return info; + } + } +} diff --git a/src/StellaOps.Feedser.Models/CvssMetric.cs b/src/StellaOps.Feedser.Models/CvssMetric.cs new file mode 100644 index 00000000..492db38e --- /dev/null +++ b/src/StellaOps.Feedser.Models/CvssMetric.cs @@ -0,0 +1,31 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Models; + +/// +/// Canonicalized CVSS metric details supporting deterministic serialization. +/// +public sealed record CvssMetric +{ + public static CvssMetric Empty { get; } = new("3.1", vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:N", baseScore: 0, baseSeverity: "none", provenance: AdvisoryProvenance.Empty); + + [JsonConstructor] + public CvssMetric(string version, string vector, double baseScore, string baseSeverity, AdvisoryProvenance provenance) + { + Version = Validation.EnsureNotNullOrWhiteSpace(version, nameof(version)); + Vector = Validation.EnsureNotNullOrWhiteSpace(vector, nameof(vector)); + BaseSeverity = Validation.EnsureNotNullOrWhiteSpace(baseSeverity, nameof(baseSeverity)).ToLowerInvariant(); + BaseScore = Math.Round(baseScore, 1, MidpointRounding.AwayFromZero); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + public string Version { get; } + + public string Vector { get; } + + public double BaseScore { get; } + + public string BaseSeverity { get; } + + public AdvisoryProvenance Provenance { get; } +} diff --git a/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md b/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md new file mode 100644 index 00000000..2536cafa --- /dev/null +++ b/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md @@ -0,0 +1,12 @@ +# Canonical Field Provenance Guidelines + +- **Always attach provenance** when mapping any field into `StellaOps.Feedser.Models`. Use `AdvisoryProvenance` to capture `source` (feed identifier), `kind` (fetch|parse|map|merge), `value` (cursor or extractor hint), and the UTC timestamp when it was recorded. +- **Per-field strategy** + - `Advisory` metadata (title, summary, severity) should record the connector responsible for the value. When merge overrides occur, add an additional provenance record rather than mutating the original. + - `References` must record whether the link originated from the primary advisory (`kind=advisory`), a vendor patch (`kind=patch`), or an enrichment feed (`kind=enrichment`). + - `AffectedPackage` records should capture the exact extraction routine (e.g., `map:oval`, `map:nvd`, `map:vendor`). + - `CvssMetric` provenance should include the scoring authority (e.g., `nvd`, `redhat`) and whether it was supplied or derived. + - `AffectedVersionRange` provenance anchors the transcript used to build the range. Preserve version strings as given by the source to aid debugging. +- **Merge policy**: never discard provenance when merging; instead append a new `AdvisoryProvenance` entry with the merge routine (`source=merge.determine-precedence`). +- **Determinism**: provenance collections are sorted by source → kind → recordedAt before serialization; avoid generating random identifiers inside provenance. +- **Redaction**: keep provenance values free of secrets; prefer tokens or normalized descriptors when referencing authenticated fetches. diff --git a/src/StellaOps.Feedser.Models/SeverityNormalization.cs b/src/StellaOps.Feedser.Models/SeverityNormalization.cs new file mode 100644 index 00000000..7c2250e1 --- /dev/null +++ b/src/StellaOps.Feedser.Models/SeverityNormalization.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Models; + +/// +/// Provides helpers to normalize vendor-provided severity labels into canonical values. +/// +public static class SeverityNormalization +{ + private static readonly IReadOnlyDictionary SeverityMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["critical"] = "critical", + ["sev_critical"] = "critical", + ["sev-critical"] = "critical", + ["high"] = "high", + ["sev_high"] = "high", + ["sev-high"] = "high", + ["important"] = "high", + ["severe"] = "high", + ["medium"] = "medium", + ["moderate"] = "medium", + ["normal"] = "medium", + ["avg"] = "medium", + ["low"] = "low", + ["minor"] = "low", + ["info"] = "informational", + ["informational"] = "informational", + ["notice"] = "informational", + ["none"] = "none", + ["negligible"] = "none", + ["unknown"] = "unknown", + }; + + public static readonly IReadOnlyCollection CanonicalLevels = new[] + { + "critical", + "high", + "medium", + "low", + "informational", + "none", + "unknown", + }; + + public static string? Normalize(string? severity) + { + if (string.IsNullOrWhiteSpace(severity)) + { + return null; + } + + var trimmed = severity.Trim(); + if (SeverityMap.TryGetValue(trimmed, out var mapped)) + { + return mapped; + } + + var lowered = trimmed.ToLowerInvariant(); + + if (SeverityMap.TryGetValue(lowered, out mapped)) + { + return mapped; + } + + return lowered; + } +} diff --git a/src/StellaOps.Feedser.Models/SnapshotSerializer.cs b/src/StellaOps.Feedser.Models/SnapshotSerializer.cs new file mode 100644 index 00000000..795ee103 --- /dev/null +++ b/src/StellaOps.Feedser.Models/SnapshotSerializer.cs @@ -0,0 +1,27 @@ +using System.Text; +using System.Text.Json; + +namespace StellaOps.Feedser.Models; + +/// +/// Helper for tests/fixtures that need deterministic JSON snapshots. +/// +public static class SnapshotSerializer +{ + public static string ToSnapshot(T value) + => CanonicalJsonSerializer.SerializeIndented(value); + + public static void AppendSnapshot(StringBuilder builder, T value) + { + ArgumentNullException.ThrowIfNull(builder); + builder.AppendLine(ToSnapshot(value)); + } + + public static async Task WriteSnapshotAsync(Stream destination, T value, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(destination); + await using var writer = new StreamWriter(destination, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), leaveOpen: true); + await writer.WriteAsync(ToSnapshot(value).AsMemory(), cancellationToken).ConfigureAwait(false); + await writer.FlushAsync().ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj b/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj new file mode 100644 index 00000000..ecc3af66 --- /dev/null +++ b/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj @@ -0,0 +1,9 @@ + + + net10.0 + preview + enable + enable + true + + diff --git a/src/StellaOps.Feedser.Models/TASKS.md b/src/StellaOps.Feedser.Models/TASKS.md new file mode 100644 index 00000000..2ded64ca --- /dev/null +++ b/src/StellaOps.Feedser.Models/TASKS.md @@ -0,0 +1,18 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Canonical JSON serializer with stable ordering|BE-Merge|Models|DONE – `CanonicalJsonSerializer` ensures deterministic property ordering.| +|Equality/comparison helpers for ranges|BE-Merge|Models|DONE – added `AffectedVersionRangeComparer` & equality comparer.| +|Type enums/constants for AffectedPackage.Type|BE-Merge|Models|DONE – introduced `AffectedPackageTypes`.| +|Validation helpers (lightweight)|BE-Merge|Models|DONE – added `Validation` static helpers and URL guard.| +|Snapshot serializer for tests|QA|Models|DONE – `SnapshotSerializer` emits canonical JSON.| +|Docs: field provenance guidelines|BE-Merge|Models|DONE – see `PROVENANCE_GUIDELINES.md`.| +|Canonical record definitions kept in sync|BE-Merge|Models|DONE – documented in `CANONICAL_RECORDS.md`; update alongside model changes.| +|Alias scheme registry and validation helpers|BE-Merge|Models|DONE – see `AliasSchemes` & `AliasSchemeRegistry` plus validation integration/tests.| +|Range primitives for SemVer/EVR/NEVRA metadata|BE-Merge|Models|TODO – keep structured values without parsing logic; ensure merge/export parity.| +|Provenance envelope field masks|BE-Merge|Models|TODO – guarantee traceability for each mapped field.| +|Backward-compatibility playbook|BE-Merge, QA|Models|DONE – see `BACKWARD_COMPATIBILITY.md` for evolution policy/test checklist.| +|Golden canonical examples|QA|Models|DOING – `CanonicalExampleFactory` + tests added; fixtures need regeneration hookup to enable easy updates.| +|Serialization determinism regression tests|QA|Models|TODO – automate hash comparisons across locales/runs.| +|Severity normalization helpers|BE-Merge|Models|DOING – helper introduced to normalize vendor severities; refine synonym coverage.| +|AffectedPackage status glossary & guardrails|BE-Merge|Models|DOING – status catalog with validation added; monitor for additional vendor values.| diff --git a/src/StellaOps.Feedser.Models/Validation.cs b/src/StellaOps.Feedser.Models/Validation.cs new file mode 100644 index 00000000..9b586ca1 --- /dev/null +++ b/src/StellaOps.Feedser.Models/Validation.cs @@ -0,0 +1,57 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Models; + +/// +/// Lightweight validation helpers shared across canonical model constructors. +/// +public static partial class Validation +{ + public static string EnsureNotNullOrWhiteSpace(string value, string paramName) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException($"Value cannot be null or whitespace.", paramName); + } + + return value.Trim(); + } + + public static string? TrimToNull(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + public static bool LooksLikeHttpUrl(string? value) + => value is not null && Uri.TryCreate(value, UriKind.Absolute, out var uri) && (uri.Scheme is "http" or "https"); + + public static bool TryNormalizeAlias(string? value, [NotNullWhen(true)] out string? normalized) + { + normalized = TrimToNull(value); + if (normalized is null) + { + return false; + } + + if (AliasSchemeRegistry.TryNormalize(normalized, out var canonical, out _)) + { + normalized = canonical; + } + + return true; + } + + public static bool TryNormalizeIdentifier(string? value, [NotNullWhen(true)] out string? normalized) + { + normalized = TrimToNull(value); + return normalized is not null; + } + + [GeneratedRegex(@"\s+")] + private static partial Regex CollapseWhitespaceRegex(); + + public static string CollapseWhitespace(string value) + { + ArgumentNullException.ThrowIfNull(value); + return CollapseWhitespaceRegex().Replace(value, " ").Trim(); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs b/src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs new file mode 100644 index 00000000..18021f33 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs @@ -0,0 +1,70 @@ +using StellaOps.Feedser.Normalization.Identifiers; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class CpeNormalizerTests +{ + [Fact] + public void TryNormalizeCpe_Preserves2Dot3Format() + { + var input = "cpe:2.3:A:Example:Product:1.0:*:*:*:*:*:*:*"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal("cpe:2.3:a:example:product:1.0:*:*:*:*:*:*:*", normalized); + } + + [Fact] + public void TryNormalizeCpe_UpgradesUriBinding() + { + var input = "cpe:/o:RedHat:Enterprise_Linux:8"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", normalized); + } + + [Fact] + public void TryNormalizeCpe_InvalidInputReturnsFalse() + { + var success = IdentifierNormalizer.TryNormalizeCpe("not-a-cpe", out var normalized); + + Assert.False(success); + Assert.Null(normalized); + } + + [Fact] + public void TryNormalizeCpe_DecodesPercentEncodingAndEscapes() + { + var input = "cpe:/a:Example%20Corp:Widget%2fSuite:1.0:update:%7e:%2a"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal(@"cpe:2.3:a:example\ corp:widget\/suite:1.0:update:*:*:*:*:*:*", normalized); + } + + [Fact] + public void TryNormalizeCpe_ExpandsEditionFields() + { + var input = "cpe:/a:Vendor:Product:1.0:update:~pro~~windows~~:en-US"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal("cpe:2.3:a:vendor:product:1.0:update:*:en-us:pro:*:windows:*", normalized); + } + + [Fact] + public void TryNormalizeCpe_PreservesEscapedCharactersIn23() + { + var input = @"cpe:2.3:a:example:printer\/:1.2.3:*:*:*:*:*:*:*"; + + var success = IdentifierNormalizer.TryNormalizeCpe(input, out var normalized); + + Assert.True(success); + Assert.Equal(@"cpe:2.3:a:example:printer\/:1.2.3:*:*:*:*:*:*:*", normalized); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs b/src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs new file mode 100644 index 00000000..038a2943 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs @@ -0,0 +1,52 @@ +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Cvss; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class CvssMetricNormalizerTests +{ + [Fact] + public void TryNormalize_ComputesCvss31Defaults() + { + var vector = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"; + + var success = CvssMetricNormalizer.TryNormalize(null, vector, null, null, out var normalized); + + Assert.True(success); + Assert.Equal("3.1", normalized.Version); + Assert.Equal(vector, normalized.Vector); + Assert.Equal(9.8, normalized.BaseScore); + Assert.Equal("critical", normalized.BaseSeverity); + + var provenance = new AdvisoryProvenance("nvd", "cvss", "https://example", DateTimeOffset.UnixEpoch); + var metric = normalized.ToModel(provenance); + Assert.Equal("3.1", metric.Version); + Assert.Equal(vector, metric.Vector); + Assert.Equal(9.8, metric.BaseScore); + Assert.Equal("critical", metric.BaseSeverity); + Assert.Equal(provenance, metric.Provenance); + } + + [Fact] + public void TryNormalize_NormalizesCvss20Severity() + { + var vector = "AV:N/AC:M/Au:S/C:P/I:P/A:P"; + + var success = CvssMetricNormalizer.TryNormalize("2.0", vector, 6.4, "MEDIUM", out var normalized); + + Assert.True(success); + Assert.Equal("2.0", normalized.Version); + Assert.Equal("CVSS:2.0/AV:N/AC:M/AU:S/C:P/I:P/A:P", normalized.Vector); + Assert.Equal(6.0, normalized.BaseScore); + Assert.Equal("medium", normalized.BaseSeverity); + } + + [Fact] + public void TryNormalize_ReturnsFalseWhenVectorMissing() + { + var success = CvssMetricNormalizer.TryNormalize("3.1", string.Empty, 9.8, "CRITICAL", out var normalized); + + Assert.False(success); + Assert.Equal(default, normalized); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs b/src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs new file mode 100644 index 00000000..e1de86c7 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs @@ -0,0 +1,31 @@ +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class DebianEvrParserTests +{ + [Fact] + public void ToCanonicalString_RoundTripsExplicitEpoch() + { + var parsed = DebianEvr.Parse(" 1:1.2.3-1 "); + + Assert.Equal("1:1.2.3-1", parsed.Original); + Assert.Equal("1:1.2.3-1", parsed.ToCanonicalString()); + } + + [Fact] + public void ToCanonicalString_SuppressesZeroEpochWhenMissing() + { + var parsed = DebianEvr.Parse("1.2.3-1"); + + Assert.Equal("1.2.3-1", parsed.ToCanonicalString()); + } + + [Fact] + public void ToCanonicalString_HandlesMissingRevision() + { + var parsed = DebianEvr.Parse("2:4.5"); + + Assert.Equal("2:4.5", parsed.ToCanonicalString()); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs b/src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs new file mode 100644 index 00000000..e04f5098 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs @@ -0,0 +1,44 @@ +using StellaOps.Feedser.Normalization.Text; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class DescriptionNormalizerTests +{ + [Fact] + public void Normalize_RemovesMarkupAndCollapsesWhitespace() + { + var candidates = new[] + { + new LocalizedText("

    Hello\n\nworld!

    ", "en-US"), + }; + + var result = DescriptionNormalizer.Normalize(candidates); + + Assert.Equal("hello world!", result.Text.ToLowerInvariant()); + Assert.Equal("en", result.Language); + } + + [Fact] + public void Normalize_FallsBackToPreferredLanguage() + { + var candidates = new[] + { + new LocalizedText("Bonjour", "fr"), + new LocalizedText("Hello", "en-GB"), + }; + + var result = DescriptionNormalizer.Normalize(candidates); + + Assert.Equal("Hello", result.Text); + Assert.Equal("en", result.Language); + } + + [Fact] + public void Normalize_ReturnsDefaultWhenEmpty() + { + var result = DescriptionNormalizer.Normalize(Array.Empty()); + + Assert.Equal(string.Empty, result.Text); + Assert.Equal("en", result.Language); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs b/src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs new file mode 100644 index 00000000..3fa48927 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs @@ -0,0 +1,64 @@ +using StellaOps.Feedser.Normalization.Distro; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class NevraParserTests +{ + [Fact] + public void ToCanonicalString_RoundTripsTrimmedInput() + { + var parsed = Nevra.Parse(" kernel-0:4.18.0-80.el8.x86_64 "); + + Assert.Equal("kernel-0:4.18.0-80.el8.x86_64", parsed.Original); + Assert.Equal("kernel-0:4.18.0-80.el8.x86_64", parsed.ToCanonicalString()); + } + + [Fact] + public void ToCanonicalString_ReconstructsKnownArchitecture() + { + var parsed = Nevra.Parse("bash-5.2.15-3.el9_4.arm64"); + + Assert.Equal("bash-5.2.15-3.el9_4.arm64", parsed.ToCanonicalString()); + } + + [Fact] + public void ToCanonicalString_HandlesMissingArchitecture() + { + var parsed = Nevra.Parse("openssl-libs-1:1.1.1k-7.el8"); + + Assert.Equal("openssl-libs-1:1.1.1k-7.el8", parsed.ToCanonicalString()); + } + + [Fact] + public void TryParse_ReturnsTrueForExplicitZeroEpoch() + { + var success = Nevra.TryParse("glibc-0:2.36-8.el9.x86_64", out var nevra); + + Assert.True(success); + Assert.NotNull(nevra); + Assert.True(nevra!.HasExplicitEpoch); + Assert.Equal(0, nevra.Epoch); + Assert.Equal("glibc-0:2.36-8.el9.x86_64", nevra.ToCanonicalString()); + } + + [Fact] + public void TryParse_IgnoresUnknownArchitectureSuffix() + { + var success = Nevra.TryParse("package-1.0-1.el9.weirdarch", out var nevra); + + Assert.True(success); + Assert.NotNull(nevra); + Assert.Null(nevra!.Architecture); + Assert.Equal("package-1.0-1.el9.weirdarch", nevra.Original); + Assert.Equal("package-1.0-1.el9.weirdarch", nevra.ToCanonicalString()); + } + + [Fact] + public void TryParse_ReturnsFalseForMalformedNevra() + { + var success = Nevra.TryParse("bad-format", out var nevra); + + Assert.False(success); + Assert.Null(nevra); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs b/src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs new file mode 100644 index 00000000..b754f6e3 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs @@ -0,0 +1,44 @@ +using System.Linq; +using StellaOps.Feedser.Normalization.Identifiers; + +namespace StellaOps.Feedser.Normalization.Tests; + +public sealed class PackageUrlNormalizerTests +{ + [Fact] + public void TryNormalizePackageUrl_LowersTypeAndNamespace() + { + var input = "pkg:NPM/Acme/Widget@1.0.0?Arch=X86_64"; + + var success = IdentifierNormalizer.TryNormalizePackageUrl(input, out var normalized, out var parsed); + + Assert.True(success); + Assert.Equal("pkg:npm/acme/widget@1.0.0?arch=X86_64", normalized); + Assert.NotNull(parsed); + Assert.Equal("npm", parsed!.Type); + Assert.Equal(new[] { "acme" }, parsed.NamespaceSegments.ToArray()); + Assert.Equal("widget", parsed.Name); + } + + [Fact] + public void TryNormalizePackageUrl_OrdersQualifiers() + { + var input = "pkg:deb/debian/openssl?distro=x%2Fy&arch=amd64"; + + var success = IdentifierNormalizer.TryNormalizePackageUrl(input, out var normalized, out _); + + Assert.True(success); + Assert.Equal("pkg:deb/debian/openssl?arch=amd64&distro=x%2Fy", normalized); + } + + [Fact] + public void TryNormalizePackageUrl_TrimsWhitespace() + { + var input = " pkg:pypi/Example/Package "; + + var success = IdentifierNormalizer.TryNormalizePackageUrl(input, out var normalized, out _); + + Assert.True(success); + Assert.Equal("pkg:pypi/example/package", normalized); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj b/src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj new file mode 100644 index 00000000..ed4b68a5 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Feedser.Normalization/AssemblyInfo.cs b/src/StellaOps.Feedser.Normalization/AssemblyInfo.cs new file mode 100644 index 00000000..b3d20fa8 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/AssemblyInfo.cs @@ -0,0 +1,8 @@ +using System.Reflection; + +[assembly: AssemblyCompany("StellaOps")] +[assembly: AssemblyProduct("StellaOps.Feedser.Normalization")] +[assembly: AssemblyTitle("StellaOps.Feedser.Normalization")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] +[assembly: AssemblyInformationalVersion("1.0.0")] diff --git a/src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs b/src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs new file mode 100644 index 00000000..b9ce4ea1 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs @@ -0,0 +1,529 @@ +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Normalization.Cvss; + +/// +/// Provides helpers to canonicalize CVSS vectors and fill in derived score/severity information. +/// +public static class CvssMetricNormalizer +{ + private static readonly string[] Cvss3BaseMetrics = { "AV", "AC", "PR", "UI", "S", "C", "I", "A" }; + private static readonly string[] Cvss2BaseMetrics = { "AV", "AC", "AU", "C", "I", "A" }; + + public static bool TryNormalize( + string? version, + string? vector, + double? baseScore, + string? baseSeverity, + out CvssNormalizedMetric metric) + { + metric = default; + if (string.IsNullOrWhiteSpace(vector)) + { + return false; + } + + var rawVector = vector.Trim(); + if (!TryDetermineVersion(version, rawVector, out var parsedVersion, out var vectorWithoutPrefix)) + { + return false; + } + + if (!TryParseMetrics(vectorWithoutPrefix, parsedVersion, out var canonicalVector, out var metrics)) + { + return false; + } + + if (!TryComputeBaseScore(parsedVersion, metrics, out var computedScore)) + { + return false; + } + + var normalizedScore = baseScore.HasValue + ? Math.Round(baseScore.Value, 1, MidpointRounding.AwayFromZero) + : computedScore; + + if (baseScore.HasValue && Math.Abs(normalizedScore - computedScore) > 0.2) + { + normalizedScore = computedScore; + } + + var severity = NormalizeSeverity(baseSeverity, parsedVersion) + ?? DetermineSeverity(normalizedScore, parsedVersion); + + metric = new CvssNormalizedMetric( + ToVersionString(parsedVersion), + canonicalVector, + normalizedScore, + severity); + + return true; + } + + private static bool TryDetermineVersion(string? versionToken, string vector, out CvssVersion version, out string withoutPrefix) + { + if (TryExtractVersionFromVector(vector, out version, out withoutPrefix)) + { + return true; + } + + if (!string.IsNullOrWhiteSpace(versionToken) && TryMapVersion(versionToken!, out version)) + { + withoutPrefix = StripPrefix(vector); + return true; + } + + var upper = vector.ToUpperInvariant(); + if (upper.Contains("PR:", StringComparison.Ordinal)) + { + version = CvssVersion.V31; + withoutPrefix = StripPrefix(vector); + return true; + } + + if (upper.Contains("AU:", StringComparison.Ordinal)) + { + version = CvssVersion.V20; + withoutPrefix = StripPrefix(vector); + return true; + } + + version = CvssVersion.V31; + withoutPrefix = StripPrefix(vector); + return true; + } + + private static string StripPrefix(string vector) + { + if (!vector.StartsWith("CVSS:", StringComparison.OrdinalIgnoreCase)) + { + return vector; + } + + var remainder = vector[5..]; + var slashIndex = remainder.IndexOf('/'); + return slashIndex >= 0 && slashIndex < remainder.Length - 1 + ? remainder[(slashIndex + 1)..] + : string.Empty; + } + + private static bool TryExtractVersionFromVector(string vector, out CvssVersion version, out string withoutPrefix) + { + withoutPrefix = vector; + if (!vector.StartsWith("CVSS:", StringComparison.OrdinalIgnoreCase)) + { + version = default; + return false; + } + + var remainder = vector[5..]; + var slashIndex = remainder.IndexOf('/'); + if (slashIndex <= 0 || slashIndex >= remainder.Length - 1) + { + version = CvssVersion.V31; + withoutPrefix = slashIndex > 0 && slashIndex < remainder.Length - 1 + ? remainder[(slashIndex + 1)..] + : string.Empty; + return false; + } + + var versionToken = remainder[..slashIndex]; + withoutPrefix = remainder[(slashIndex + 1)..]; + if (TryMapVersion(versionToken, out version)) + { + return true; + } + + version = CvssVersion.V31; + return false; + } + + private static bool TryMapVersion(string token, out CvssVersion version) + { + var trimmed = token.Trim(); + if (trimmed.Length == 0) + { + version = default; + return false; + } + + if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed[1..]; + } + + trimmed = trimmed switch + { + "3" or "3.1.0" or "3.1" => "3.1", + "3.0" or "3.0.0" => "3.0", + "2" or "2.0.0" => "2.0", + _ => trimmed, + }; + + version = trimmed switch + { + "2" or "2.0" => CvssVersion.V20, + "3.0" => CvssVersion.V30, + "3.1" => CvssVersion.V31, + _ => CvssVersion.Unknown, + }; + + return version != CvssVersion.Unknown; + } + + private static bool TryParseMetrics( + string vector, + CvssVersion version, + out string canonicalVector, + out ImmutableDictionary metrics) + { + canonicalVector = string.Empty; + var parsed = new Dictionary(StringComparer.OrdinalIgnoreCase); + var segments = vector.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length == 0) + { + metrics = ImmutableDictionary.Empty; + return false; + } + + foreach (var segment in segments) + { + var trimmed = segment.Trim(); + if (trimmed.Length == 0) + { + continue; + } + + var index = trimmed.IndexOf(':'); + if (index <= 0 || index == trimmed.Length - 1) + { + metrics = ImmutableDictionary.Empty; + return false; + } + + var key = trimmed[..index].Trim().ToUpperInvariant(); + var value = trimmed[(index + 1)..].Trim().ToUpperInvariant(); + if (key.Length == 0 || value.Length == 0) + { + metrics = ImmutableDictionary.Empty; + return false; + } + + parsed[key] = value; + } + + var required = version == CvssVersion.V20 ? Cvss2BaseMetrics : Cvss3BaseMetrics; + foreach (var metric in required) + { + if (!parsed.ContainsKey(metric)) + { + metrics = ImmutableDictionary.Empty; + return false; + } + } + + var canonicalSegments = new List(parsed.Count + 1); + foreach (var metric in required) + { + canonicalSegments.Add($"{metric}:{parsed[metric]}"); + } + + foreach (var entry in parsed.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + if (required.Contains(entry.Key)) + { + continue; + } + + canonicalSegments.Add($"{entry.Key}:{entry.Value}"); + } + + canonicalVector = $"CVSS:{ToVersionString(version)}/{string.Join('/', canonicalSegments)}"; + metrics = parsed.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + return true; + } + + private static bool TryComputeBaseScore(CvssVersion version, IReadOnlyDictionary metrics, out double score) + { + return version switch + { + CvssVersion.V20 => TryComputeCvss2(metrics, out score), + CvssVersion.V30 or CvssVersion.V31 => TryComputeCvss3(metrics, out score), + _ => (score = 0) == 0, + }; + } + + private static bool TryComputeCvss3(IReadOnlyDictionary metrics, out double score) + { + try + { + var av = metrics["AV"] switch + { + "N" => 0.85, + "A" => 0.62, + "L" => 0.55, + "P" => 0.2, + _ => throw new InvalidOperationException(), + }; + + var ac = metrics["AC"] switch + { + "L" => 0.77, + "H" => 0.44, + _ => throw new InvalidOperationException(), + }; + + var scopeChanged = metrics["S"] switch + { + "U" => false, + "C" => true, + _ => throw new InvalidOperationException(), + }; + + var pr = metrics["PR"] switch + { + "N" => 0.85, + "L" => scopeChanged ? 0.68 : 0.62, + "H" => scopeChanged ? 0.5 : 0.27, + _ => throw new InvalidOperationException(), + }; + + var ui = metrics["UI"] switch + { + "N" => 0.85, + "R" => 0.62, + _ => throw new InvalidOperationException(), + }; + + var confidentiality = metrics["C"] switch + { + "N" => 0.0, + "L" => 0.22, + "H" => 0.56, + _ => throw new InvalidOperationException(), + }; + + var integrity = metrics["I"] switch + { + "N" => 0.0, + "L" => 0.22, + "H" => 0.56, + _ => throw new InvalidOperationException(), + }; + + var availability = metrics["A"] switch + { + "N" => 0.0, + "L" => 0.22, + "H" => 0.56, + _ => throw new InvalidOperationException(), + }; + + var impactSub = 1 - (1 - confidentiality) * (1 - integrity) * (1 - availability); + impactSub = Math.Clamp(impactSub, 0, 1); + + var impact = scopeChanged + ? 7.52 * (impactSub - 0.029) - 3.25 * Math.Pow(impactSub - 0.02, 15) + : 6.42 * impactSub; + + var exploitability = 8.22 * av * ac * pr * ui; + + if (impact <= 0) + { + score = 0; + return true; + } + + var baseScore = scopeChanged + ? Math.Min(1.08 * (impact + exploitability), 10) + : Math.Min(impact + exploitability, 10); + + score = RoundUp(baseScore); + return true; + } + catch (KeyNotFoundException) + { + score = 0; + return false; + } + catch (InvalidOperationException) + { + score = 0; + return false; + } + } + + private static bool TryComputeCvss2(IReadOnlyDictionary metrics, out double score) + { + try + { + var av = metrics["AV"] switch + { + "L" => 0.395, + "A" => 0.646, + "N" => 1.0, + _ => throw new InvalidOperationException(), + }; + + var ac = metrics["AC"] switch + { + "H" => 0.35, + "M" => 0.61, + "L" => 0.71, + _ => throw new InvalidOperationException(), + }; + + var authValue = metrics.TryGetValue("AU", out var primaryAuth) + ? primaryAuth + : metrics.TryGetValue("AUTH", out var fallbackAuth) + ? fallbackAuth + : null; + + if (string.IsNullOrEmpty(authValue)) + { + throw new InvalidOperationException(); + } + + var authentication = authValue switch + { + "M" => 0.45, + "S" => 0.56, + "N" => 0.704, + _ => throw new InvalidOperationException(), + }; + + var confidentiality = metrics["C"] switch + { + "N" => 0.0, + "P" => 0.275, + "C" => 0.660, + _ => throw new InvalidOperationException(), + }; + + var integrity = metrics["I"] switch + { + "N" => 0.0, + "P" => 0.275, + "C" => 0.660, + _ => throw new InvalidOperationException(), + }; + + var availability = metrics["A"] switch + { + "N" => 0.0, + "P" => 0.275, + "C" => 0.660, + _ => throw new InvalidOperationException(), + }; + + var impact = 10.41 * (1 - (1 - confidentiality) * (1 - integrity) * (1 - availability)); + var exploitability = 20 * av * ac * authentication; + var fImpact = impact == 0 ? 0.0 : 1.176; + var baseScore = ((0.6 * impact) + (0.4 * exploitability) - 1.5) * fImpact; + score = Math.Round(Math.Max(baseScore, 0), 1, MidpointRounding.AwayFromZero); + return true; + } + catch (KeyNotFoundException) + { + score = 0; + return false; + } + catch (InvalidOperationException) + { + score = 0; + return false; + } + } + + private static string DetermineSeverity(double score, CvssVersion version) + { + if (score <= 0) + { + return "none"; + } + + if (version == CvssVersion.V20) + { + if (score < 4.0) + { + return "low"; + } + + if (score < 7.0) + { + return "medium"; + } + + return "high"; + } + + if (score < 4.0) + { + return "low"; + } + + if (score < 7.0) + { + return "medium"; + } + + if (score < 9.0) + { + return "high"; + } + + return "critical"; + } + + private static string? NormalizeSeverity(string? severity, CvssVersion version) + { + if (string.IsNullOrWhiteSpace(severity)) + { + return null; + } + + var normalized = severity.Trim().ToLowerInvariant(); + return normalized switch + { + "none" or "informational" or "info" => "none", + "critical" when version != CvssVersion.V20 => "critical", + "critical" when version == CvssVersion.V20 => "high", + "high" => "high", + "medium" or "moderate" => "medium", + "low" => "low", + _ => null, + }; + } + + private static double RoundUp(double value) + { + return Math.Ceiling(value * 10.0) / 10.0; + } + + private static string ToVersionString(CvssVersion version) + => version switch + { + CvssVersion.V20 => "2.0", + CvssVersion.V30 => "3.0", + _ => "3.1", + }; + + private enum CvssVersion + { + Unknown = 0, + V20, + V30, + V31, + } +} + +/// +/// Represents a normalized CVSS metric ready for canonical serialization. +/// +public readonly record struct CvssNormalizedMetric(string Version, string Vector, double BaseScore, string BaseSeverity) +{ + public CvssMetric ToModel(AdvisoryProvenance provenance) + => new(Version, Vector, BaseScore, BaseSeverity, provenance); +} diff --git a/src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs b/src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs new file mode 100644 index 00000000..e47c3f76 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs @@ -0,0 +1,127 @@ +using System.Globalization; + +namespace StellaOps.Feedser.Normalization.Distro; + +/// +/// Represents a Debian epoch:version-revision tuple and exposes parsing/formatting helpers. +/// +public sealed class DebianEvr +{ + private DebianEvr(int epoch, bool hasExplicitEpoch, string version, string revision, string original) + { + Epoch = epoch; + HasExplicitEpoch = hasExplicitEpoch; + Version = version; + Revision = revision; + Original = original; + } + + /// + /// Epoch segment (defaults to 0 when omitted). + /// + public int Epoch { get; } + + /// + /// Indicates whether an epoch segment was present explicitly. + /// + public bool HasExplicitEpoch { get; } + + /// + /// Version portion (without revision). + /// + public string Version { get; } + + /// + /// Revision portion (after the last dash). Empty when omitted. + /// + public string Revision { get; } + + /// + /// Trimmed EVR string supplied to . + /// + public string Original { get; } + + /// + /// Attempts to parse the provided value into a instance. + /// + public static bool TryParse(string? value, out DebianEvr? result) + { + result = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var epoch = 0; + var hasExplicitEpoch = false; + var remainder = trimmed; + + var colonIndex = remainder.IndexOf(':'); + if (colonIndex >= 0) + { + if (colonIndex == 0) + { + return false; + } + + var epochPart = remainder[..colonIndex]; + if (!int.TryParse(epochPart, NumberStyles.Integer, CultureInfo.InvariantCulture, out epoch)) + { + return false; + } + + hasExplicitEpoch = true; + remainder = colonIndex < remainder.Length - 1 ? remainder[(colonIndex + 1)..] : string.Empty; + } + + if (string.IsNullOrEmpty(remainder)) + { + return false; + } + + var version = remainder; + var revision = string.Empty; + + var dashIndex = remainder.LastIndexOf('-'); + if (dashIndex > 0) + { + version = remainder[..dashIndex]; + revision = dashIndex < remainder.Length - 1 ? remainder[(dashIndex + 1)..] : string.Empty; + } + + if (string.IsNullOrEmpty(version)) + { + return false; + } + + result = new DebianEvr(epoch, hasExplicitEpoch, version, revision, trimmed); + return true; + } + + /// + /// Parses the provided value into a or throws . + /// + public static DebianEvr Parse(string value) + { + if (!TryParse(value, out var evr)) + { + throw new FormatException($"Input '{value}' is not a valid Debian EVR string."); + } + + return evr!; + } + + /// + /// Returns a canonical EVR string with trimmed components and normalized epoch/revision placement. + /// + public string ToCanonicalString() + { + var epochSegment = HasExplicitEpoch || Epoch > 0 ? $"{Epoch}:" : string.Empty; + var revisionSegment = string.IsNullOrEmpty(Revision) ? string.Empty : $"-{Revision}"; + return $"{epochSegment}{Version}{revisionSegment}"; + } + + /// + public override string ToString() => Original; +} diff --git a/src/StellaOps.Feedser.Normalization/Distro/Nevra.cs b/src/StellaOps.Feedser.Normalization/Distro/Nevra.cs new file mode 100644 index 00000000..14ebe0e5 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Distro/Nevra.cs @@ -0,0 +1,192 @@ +using System.Globalization; + +namespace StellaOps.Feedser.Normalization.Distro; + +/// +/// Represents a parsed NEVRA (Name-Epoch:Version-Release.Architecture) identifier and exposes helpers for canonical formatting. +/// +public sealed class Nevra +{ + private Nevra(string name, int epoch, bool hasExplicitEpoch, string version, string release, string? architecture, string original) + { + Name = name; + Epoch = epoch; + HasExplicitEpoch = hasExplicitEpoch; + Version = version; + Release = release; + Architecture = architecture; + Original = original; + } + + /// + /// Package name segment. + /// + public string Name { get; } + + /// + /// Epoch extracted from the NEVRA string (defaults to 0 when omitted). + /// + public int Epoch { get; } + + /// + /// Indicates whether an epoch segment was present explicitly (e.g. 0:). + /// + public bool HasExplicitEpoch { get; } + + /// + /// Version component (without epoch or release). + /// + public string Version { get; } + + /// + /// Release component (without architecture suffix). + /// + public string Release { get; } + + /// + /// Optional architecture suffix (e.g. x86_64, noarch). + /// + public string? Architecture { get; } + + /// + /// Trimmed NEVRA string supplied to . + /// + public string Original { get; } + + private static readonly ISet KnownArchitectures = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "noarch", + "src", + "nosrc", + "x86_64", + "aarch64", + "armv7hl", + "armhfp", + "ppc64", + "ppc64le", + "ppc", + "s390", + "s390x", + "i386", + "i486", + "i586", + "i686", + "amd64", + "arm64", + "armv7l", + "armv6l", + "armv8l", + "armel", + "armhf", + "ia32e", + "loongarch64", + "mips", + "mips64", + "mips64le", + "mipsel", + "ppc32", + "ppc64p7", + "riscv64", + "sparc", + "sparc64" + }; + + /// + /// Attempts to parse the provided value into a instance. + /// + public static bool TryParse(string? value, out Nevra? result) + { + result = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var releaseSeparator = trimmed.LastIndexOf('-'); + if (releaseSeparator <= 0 || releaseSeparator >= trimmed.Length - 1) + { + return false; + } + + var releasePart = trimmed[(releaseSeparator + 1)..]; + var nameVersionPart = trimmed[..releaseSeparator]; + + var versionSeparator = nameVersionPart.LastIndexOf('-'); + if (versionSeparator <= 0 || versionSeparator >= nameVersionPart.Length) + { + return false; + } + + var versionPart = nameVersionPart[(versionSeparator + 1)..]; + var namePart = nameVersionPart[..versionSeparator]; + + if (string.IsNullOrWhiteSpace(namePart)) + { + return false; + } + + string? architecture = null; + var release = releasePart; + var architectureSeparator = releasePart.LastIndexOf('.'); + if (architectureSeparator > 0 && architectureSeparator < releasePart.Length - 1) + { + var possibleArch = releasePart[(architectureSeparator + 1)..]; + if (KnownArchitectures.Contains(possibleArch)) + { + architecture = possibleArch; + release = releasePart[..architectureSeparator]; + } + } + + var version = versionPart; + var epoch = 0; + var hasExplicitEpoch = false; + var epochSeparator = versionPart.IndexOf(':'); + if (epochSeparator >= 0) + { + hasExplicitEpoch = true; + var epochPart = versionPart[..epochSeparator]; + version = epochSeparator < versionPart.Length - 1 ? versionPart[(epochSeparator + 1)..] : string.Empty; + + if (epochPart.Length > 0 && !int.TryParse(epochPart, NumberStyles.Integer, CultureInfo.InvariantCulture, out epoch)) + { + return false; + } + } + + if (string.IsNullOrWhiteSpace(version)) + { + return false; + } + + result = new Nevra(namePart, epoch, hasExplicitEpoch, version, release, architecture, trimmed); + return true; + } + + /// + /// Parses the provided value into a or throws . + /// + public static Nevra Parse(string value) + { + if (!TryParse(value, out var nevra)) + { + throw new FormatException($"Input '{value}' is not a valid NEVRA string."); + } + + return nevra!; + } + + /// + /// Returns a canonical NEVRA string with trimmed components and normalized epoch/architecture placement. + /// + public string ToCanonicalString() + { + var epochSegment = HasExplicitEpoch || Epoch > 0 ? $"{Epoch}:" : string.Empty; + var archSegment = string.IsNullOrWhiteSpace(Architecture) ? string.Empty : $".{Architecture}"; + return $"{Name}-{epochSegment}{Version}-{Release}{archSegment}"; + } + + /// + public override string ToString() => Original; +} diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs b/src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs new file mode 100644 index 00000000..8fc04f60 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs @@ -0,0 +1,352 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace StellaOps.Feedser.Normalization.Identifiers; + +/// +/// Implements canonical normalization for CPE 2.3 identifiers (and URI binding conversion). +/// +internal static class Cpe23 +{ + private static readonly HashSet CharactersRequiringEscape = new(new[] + { + '\\', ':', '/', '?', '#', '[', ']', '@', '!', '$', '&', '"', '\'', '(', ')', '+', ',', ';', '=', '%', '*', + '<', '>', '|', '^', '`', '{', '}', '~' + }); + + public static bool TryNormalize(string? value, out string? normalized) + { + normalized = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var components = SplitComponents(trimmed); + if (components.Count == 0) + { + return false; + } + + if (!components[0].Equals("cpe", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (components.Count >= 2 && components[1].Equals("2.3", StringComparison.OrdinalIgnoreCase)) + { + return TryNormalizeFrom23(components, out normalized); + } + + if (components.Count >= 2 && components[1].Length > 0 && components[1][0] == '/') + { + return TryNormalizeFrom22(components, out normalized); + } + + return false; + } + + private static bool TryNormalizeFrom23(IReadOnlyList components, out string? normalized) + { + normalized = null; + if (components.Count != 13) + { + return false; + } + + var part = NormalizePart(components[2]); + if (part is null) + { + return false; + } + + var normalizedComponents = new string[13]; + normalizedComponents[0] = "cpe"; + normalizedComponents[1] = "2.3"; + normalizedComponents[2] = part; + normalizedComponents[3] = NormalizeField(components[3], lower: true, decodeUri: false); + normalizedComponents[4] = NormalizeField(components[4], lower: true, decodeUri: false); + normalizedComponents[5] = NormalizeField(components[5], lower: false, decodeUri: false); + normalizedComponents[6] = NormalizeField(components[6], lower: false, decodeUri: false); + normalizedComponents[7] = NormalizeField(components[7], lower: false, decodeUri: false); + normalizedComponents[8] = NormalizeField(components[8], lower: false, decodeUri: false); + normalizedComponents[9] = NormalizeField(components[9], lower: false, decodeUri: false); + normalizedComponents[10] = NormalizeField(components[10], lower: false, decodeUri: false); + normalizedComponents[11] = NormalizeField(components[11], lower: false, decodeUri: false); + normalizedComponents[12] = NormalizeField(components[12], lower: false, decodeUri: false); + + normalized = string.Join(':', normalizedComponents); + return true; + } + + private static bool TryNormalizeFrom22(IReadOnlyList components, out string? normalized) + { + normalized = null; + if (components.Count < 2) + { + return false; + } + + var partComponent = components[1]; + if (partComponent.Length < 2 || partComponent[0] != '/') + { + return false; + } + + var part = NormalizePart(partComponent[1..]); + if (part is null) + { + return false; + } + + var vendor = NormalizeField(components.Count > 2 ? components[2] : null, lower: true, decodeUri: true); + var product = NormalizeField(components.Count > 3 ? components[3] : null, lower: true, decodeUri: true); + var version = NormalizeField(components.Count > 4 ? components[4] : null, lower: false, decodeUri: true); + var update = NormalizeField(components.Count > 5 ? components[5] : null, lower: false, decodeUri: true); + + var (edition, swEdition, targetSw, targetHw, other) = ExpandEdition(components.Count > 6 ? components[6] : null); + var language = NormalizeField(components.Count > 7 ? components[7] : null, lower: true, decodeUri: true); + + normalized = string.Join(':', new[] + { + "cpe", + "2.3", + part, + vendor, + product, + version, + update, + edition, + language, + swEdition, + targetSw, + targetHw, + other, + }); + + return true; + } + + private static string? NormalizePart(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var token = value.Trim().ToLowerInvariant(); + return token is "a" or "o" or "h" ? token : null; + } + + private static string NormalizeField(string? value, bool lower, bool decodeUri) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "*"; + } + + var trimmed = value.Trim(); + if (trimmed is "*" or "-") + { + return trimmed; + } + + var decoded = decodeUri ? DecodeUriComponent(trimmed) : UnescapeComponent(trimmed); + if (decoded is "*" or "-") + { + return decoded; + } + + if (decoded.Length == 0) + { + return "*"; + } + + var normalized = lower ? decoded.ToLowerInvariant() : decoded; + return EscapeComponent(normalized); + } + + private static (string Edition, string SwEdition, string TargetSw, string TargetHw, string Other) ExpandEdition(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ("*", "*", "*", "*", "*"); + } + + var trimmed = value.Trim(); + if (trimmed is "*" or "-") + { + return (trimmed, "*", "*", "*", "*"); + } + + var decoded = DecodeUriComponent(trimmed); + if (!decoded.StartsWith("~", StringComparison.Ordinal)) + { + return (NormalizeDecodedField(decoded, lower: false), "*", "*", "*", "*"); + } + + var segments = decoded.Split('~'); + var swEdition = segments.Length > 1 ? NormalizeDecodedField(segments[1], lower: false) : "*"; + var targetSw = segments.Length > 2 ? NormalizeDecodedField(segments[2], lower: false) : "*"; + var targetHw = segments.Length > 3 ? NormalizeDecodedField(segments[3], lower: false) : "*"; + var other = segments.Length > 4 ? NormalizeDecodedField(segments[4], lower: false) : "*"; + + return ("*", swEdition, targetSw, targetHw, other); + } + + private static string NormalizeDecodedField(string? value, bool lower) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "*"; + } + + var trimmed = value.Trim(); + if (trimmed is "*" or "-") + { + return trimmed; + } + + var normalized = lower ? trimmed.ToLowerInvariant() : trimmed; + if (normalized is "*" or "-") + { + return normalized; + } + + return EscapeComponent(normalized); + } + + private static string UnescapeComponent(string value) + { + var builder = new StringBuilder(value.Length); + var escape = false; + foreach (var ch in value) + { + if (escape) + { + builder.Append(ch); + escape = false; + continue; + } + + if (ch == '\\') + { + escape = true; + continue; + } + + builder.Append(ch); + } + + if (escape) + { + builder.Append('\\'); + } + + return builder.ToString(); + } + + private static string EscapeComponent(string value) + { + if (value.Length == 0) + { + return value; + } + + var builder = new StringBuilder(value.Length * 2); + foreach (var ch in value) + { + if (RequiresEscape(ch)) + { + builder.Append('\\'); + } + + builder.Append(ch); + } + + return builder.ToString(); + } + + private static bool RequiresEscape(char ch) + { + if (char.IsLetterOrDigit(ch)) + { + return false; + } + + if (char.IsWhiteSpace(ch)) + { + return true; + } + + return ch switch + { + '_' or '-' or '.' => false, + // Keep wildcard markers literal only when entire component is wildcard handled earlier. + '*' => true, + _ => CharactersRequiringEscape.Contains(ch) + }; + } + + private static string DecodeUriComponent(string value) + { + var builder = new StringBuilder(value.Length); + for (var i = 0; i < value.Length; i++) + { + var ch = value[i]; + if (ch == '%' && i + 2 < value.Length && IsHex(value[i + 1]) && IsHex(value[i + 2])) + { + var hex = new string(new[] { value[i + 1], value[i + 2] }); + var decoded = (char)int.Parse(hex, NumberStyles.HexNumber, CultureInfo.InvariantCulture); + builder.Append(decoded); + i += 2; + } + else + { + builder.Append(ch); + } + } + + return builder.ToString(); + } + + private static bool IsHex(char ch) + => ch is >= '0' and <= '9' or >= 'A' and <= 'F' or >= 'a' and <= 'f'; + + private static List SplitComponents(string value) + { + var results = new List(); + var builder = new StringBuilder(); + var escape = false; + foreach (var ch in value) + { + if (escape) + { + builder.Append(ch); + escape = false; + continue; + } + + if (ch == '\\') + { + builder.Append(ch); + escape = true; + continue; + } + + if (ch == ':') + { + results.Add(builder.ToString()); + builder.Clear(); + continue; + } + + builder.Append(ch); + } + + results.Add(builder.ToString()); + return results; + } +} diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs b/src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs new file mode 100644 index 00000000..af1392f7 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs @@ -0,0 +1,32 @@ +namespace StellaOps.Feedser.Normalization.Identifiers; + +/// +/// Provides canonical normalization helpers for package identifiers. +/// +public static class IdentifierNormalizer +{ + public static bool TryNormalizePackageUrl(string? value, out string? normalized, out PackageUrl? packageUrl) + { + normalized = null; + packageUrl = null; + if (!PackageUrl.TryParse(value, out var parsed)) + { + return false; + } + + var canonical = parsed!.ToCanonicalString(); + normalized = canonical; + packageUrl = parsed; + return true; + } + + public static bool TryNormalizePackageUrl(string? value, out string? normalized) + { + return TryNormalizePackageUrl(value, out normalized, out _); + } + + public static bool TryNormalizeCpe(string? value, out string? normalized) + { + return Cpe23.TryNormalize(value, out normalized); + } +} diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs b/src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs new file mode 100644 index 00000000..fa256efb --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs @@ -0,0 +1,299 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text; + +namespace StellaOps.Feedser.Normalization.Identifiers; + +/// +/// Represents a parsed Package URL (purl) identifier with canonical string rendering. +/// +public sealed class PackageUrl +{ + private PackageUrl( + string type, + ImmutableArray namespaceSegments, + string name, + string? version, + ImmutableArray> qualifiers, + ImmutableArray subpathSegments, + string original) + { + Type = type; + NamespaceSegments = namespaceSegments; + Name = name; + Version = version; + Qualifiers = qualifiers; + SubpathSegments = subpathSegments; + Original = original; + } + + public string Type { get; } + + public ImmutableArray NamespaceSegments { get; } + + public string Name { get; } + + public string? Version { get; } + + public ImmutableArray> Qualifiers { get; } + + public ImmutableArray SubpathSegments { get; } + + public string Original { get; } + + private static readonly HashSet LowerCaseNamespaceTypes = new(StringComparer.OrdinalIgnoreCase) + { + "maven", + "npm", + "pypi", + "nuget", + "composer", + "gem", + "apk", + "deb", + "rpm", + "oci", + }; + + private static readonly HashSet LowerCaseNameTypes = new(StringComparer.OrdinalIgnoreCase) + { + "npm", + "pypi", + "nuget", + "composer", + "gem", + "apk", + "deb", + "rpm", + "oci", + }; + + public static bool TryParse(string? value, out PackageUrl? packageUrl) + { + packageUrl = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (!trimmed.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var remainder = trimmed[4..]; + var firstSlash = remainder.IndexOf('/'); + if (firstSlash <= 0) + { + return false; + } + + var type = remainder[..firstSlash].Trim().ToLowerInvariant(); + remainder = remainder[(firstSlash + 1)..]; + + var subpathPart = string.Empty; + var subpathIndex = remainder.IndexOf('#'); + if (subpathIndex >= 0) + { + subpathPart = remainder[(subpathIndex + 1)..]; + remainder = remainder[..subpathIndex]; + } + + var qualifierPart = string.Empty; + var qualifierIndex = remainder.IndexOf('?'); + if (qualifierIndex >= 0) + { + qualifierPart = remainder[(qualifierIndex + 1)..]; + remainder = remainder[..qualifierIndex]; + } + + string? version = null; + var versionIndex = remainder.LastIndexOf('@'); + if (versionIndex >= 0) + { + version = remainder[(versionIndex + 1)..]; + remainder = remainder[..versionIndex]; + } + + if (string.IsNullOrWhiteSpace(remainder)) + { + return false; + } + + var rawSegments = remainder.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (rawSegments.Length == 0) + { + return false; + } + + var shouldLowerNamespace = LowerCaseNamespaceTypes.Contains(type); + var shouldLowerName = LowerCaseNameTypes.Contains(type); + + var namespaceBuilder = ImmutableArray.CreateBuilder(Math.Max(0, rawSegments.Length - 1)); + for (var i = 0; i < rawSegments.Length - 1; i++) + { + var segment = Uri.UnescapeDataString(rawSegments[i].Trim()); + if (segment.Length == 0) + { + continue; + } + + if (shouldLowerNamespace) + { + segment = segment.ToLowerInvariant(); + } + + namespaceBuilder.Add(EscapePathSegment(segment)); + } + + var nameSegment = Uri.UnescapeDataString(rawSegments[^1].Trim()); + if (nameSegment.Length == 0) + { + return false; + } + + if (shouldLowerName) + { + nameSegment = nameSegment.ToLowerInvariant(); + } + + var canonicalName = EscapePathSegment(nameSegment); + var canonicalVersion = NormalizeComponent(version, escape: true, lowerCase: false); + var qualifiers = ParseQualifiers(qualifierPart); + var subpath = ParseSubpath(subpathPart); + + packageUrl = new PackageUrl( + type, + namespaceBuilder.ToImmutable(), + canonicalName, + canonicalVersion, + qualifiers, + subpath, + trimmed); + return true; + } + + public static PackageUrl Parse(string value) + { + if (!TryParse(value, out var parsed)) + { + throw new FormatException($"Input '{value}' is not a valid Package URL."); + } + + return parsed!; + } + + public string ToCanonicalString() + { + var builder = new StringBuilder("pkg:"); + builder.Append(Type); + builder.Append('/'); + + if (!NamespaceSegments.IsDefaultOrEmpty) + { + builder.Append(string.Join('/', NamespaceSegments)); + builder.Append('/'); + } + + builder.Append(Name); + + if (!string.IsNullOrEmpty(Version)) + { + builder.Append('@'); + builder.Append(Version); + } + + if (!Qualifiers.IsDefaultOrEmpty && Qualifiers.Length > 0) + { + builder.Append('?'); + builder.Append(string.Join('&', Qualifiers.Select(static kvp => $"{kvp.Key}={kvp.Value}"))); + } + + if (!SubpathSegments.IsDefaultOrEmpty && SubpathSegments.Length > 0) + { + builder.Append('#'); + builder.Append(string.Join('/', SubpathSegments)); + } + + return builder.ToString(); + } + + public override string ToString() => ToCanonicalString(); + + private static ImmutableArray> ParseQualifiers(string qualifierPart) + { + if (string.IsNullOrEmpty(qualifierPart)) + { + return ImmutableArray>.Empty; + } + + var entries = qualifierPart.Split('&', StringSplitOptions.RemoveEmptyEntries); + var map = new SortedDictionary(StringComparer.Ordinal); + foreach (var entry in entries) + { + var trimmed = entry.Trim(); + if (trimmed.Length == 0) + { + continue; + } + + var equalsIndex = trimmed.IndexOf('='); + if (equalsIndex <= 0) + { + continue; + } + + var key = Uri.UnescapeDataString(trimmed[..equalsIndex]).Trim().ToLowerInvariant(); + var valuePart = equalsIndex < trimmed.Length - 1 ? trimmed[(equalsIndex + 1)..] : string.Empty; + var value = NormalizeComponent(valuePart, escape: true, lowerCase: false); + map[key] = value; + } + + return map.Select(static kvp => new KeyValuePair(kvp.Key, kvp.Value)).ToImmutableArray(); + } + + private static ImmutableArray ParseSubpath(string subpathPart) + { + if (string.IsNullOrEmpty(subpathPart)) + { + return ImmutableArray.Empty; + } + + var segments = subpathPart.Split('/', StringSplitOptions.RemoveEmptyEntries); + var builder = ImmutableArray.CreateBuilder(segments.Length); + foreach (var raw in segments) + { + var segment = Uri.UnescapeDataString(raw.Trim()); + if (segment.Length == 0) + { + continue; + } + + builder.Add(EscapePathSegment(segment)); + } + + return builder.ToImmutable(); + } + + private static string NormalizeComponent(string? value, bool escape, bool lowerCase) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var unescaped = Uri.UnescapeDataString(value.Trim()); + if (lowerCase) + { + unescaped = unescaped.ToLowerInvariant(); + } + + return escape ? Uri.EscapeDataString(unescaped) : unescaped; + } + + private static string EscapePathSegment(string value) + { + return Uri.EscapeDataString(value); + } +} diff --git a/src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj b/src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj new file mode 100644 index 00000000..1c0f5ec9 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj @@ -0,0 +1,18 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Normalization/TASKS.md b/src/StellaOps.Feedser.Normalization/TASKS.md new file mode 100644 index 00000000..82bb26e0 --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/TASKS.md @@ -0,0 +1,8 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Canonical NEVRA/EVR parsing helpers|BE-Norm (Distro WG)|Models|DONE – `Normalization.Distro` exposes parsers + canonical formatters consumed by Merge comparers/tests.| +|PURL/CPE identifier normalization|BE-Norm (OSS WG)|Models|DONE – canonical PURL/CPE helpers feed connectors and exporter tooling.| +|CPE normalization escape handling|BE-Norm (OSS WG)|Normalization identifiers|DONE – percent-decoding, edition sub-field expansion, and deterministic escaping landed in `Cpe23` with new tests covering boundary cases.| +|CVSS metric normalization & severity bands|BE-Norm (Risk WG)|Models|DONE – `CvssMetricNormalizer` unifies vectors, recomputes scores/severities, and is wired through NVD/RedHat/JVN mappers with unit coverage.| +|Description and locale normalization pipeline|BE-Norm (I18N)|Source connectors|DONE – `DescriptionNormalizer` strips markup, collapses whitespace, and provides locale fallback used by core mappers.| diff --git a/src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs b/src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs new file mode 100644 index 00000000..08a4701b --- /dev/null +++ b/src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs @@ -0,0 +1,118 @@ +using System.Globalization; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Normalization.Text; + +/// +/// Normalizes advisory descriptions by stripping markup, collapsing whitespace, and selecting the best locale fallback. +/// +public static class DescriptionNormalizer +{ + private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly string[] PreferredLanguages = { "en", "en-us", "en-gb" }; + + public static NormalizedDescription Normalize(IEnumerable candidates) + { + if (candidates is null) + { + throw new ArgumentNullException(nameof(candidates)); + } + + var processed = new List<(string Text, string Language, int Index)>(); + var index = 0; + foreach (var candidate in candidates) + { + if (string.IsNullOrWhiteSpace(candidate.Text)) + { + index++; + continue; + } + + var sanitized = Sanitize(candidate.Text); + if (string.IsNullOrWhiteSpace(sanitized)) + { + index++; + continue; + } + + var language = NormalizeLanguage(candidate.Language); + processed.Add((sanitized, language, index)); + index++; + } + + if (processed.Count == 0) + { + return new NormalizedDescription(string.Empty, "en"); + } + + var best = SelectBest(processed); + var languageTag = best.Language.Length > 0 ? best.Language : "en"; + return new NormalizedDescription(best.Text, languageTag); + } + + private static (string Text, string Language) SelectBest(List<(string Text, string Language, int Index)> processed) + { + foreach (var preferred in PreferredLanguages) + { + var normalized = NormalizeLanguage(preferred); + var match = processed.FirstOrDefault(entry => entry.Language.Equals(normalized, StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrEmpty(match.Text)) + { + return (match.Text, normalized); + } + } + + var first = processed.OrderBy(entry => entry.Index).First(); + return (first.Text, first.Language); + } + + private static string Sanitize(string text) + { + var decoded = WebUtility.HtmlDecode(text) ?? string.Empty; + var withoutTags = HtmlTagRegex.Replace(decoded, " "); + var collapsed = WhitespaceRegex.Replace(withoutTags, " ").Trim(); + return collapsed; + } + + private static string NormalizeLanguage(string? language) + { + if (string.IsNullOrWhiteSpace(language)) + { + return string.Empty; + } + + var trimmed = language.Trim(); + try + { + var culture = CultureInfo.GetCultureInfo(trimmed); + if (!string.IsNullOrEmpty(culture.Name)) + { + var parts = culture.Name.Split('-'); + if (parts.Length > 0 && !string.IsNullOrWhiteSpace(parts[0])) + { + return parts[0].ToLowerInvariant(); + } + } + } + catch (CultureNotFoundException) + { + // fall back to manual normalization + } + + var primary = trimmed.Split(new[] { '-', '_' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + return string.IsNullOrWhiteSpace(primary) ? string.Empty : primary.ToLowerInvariant(); + } +} + +/// +/// Represents a localized text candidate. +/// +public readonly record struct LocalizedText(string? Text, string? Language); + +/// +/// Represents a normalized description result. +/// +public readonly record struct NormalizedDescription(string Text, string Language); diff --git a/src/StellaOps.Feedser.Source.Acsc/Class1.cs b/src/StellaOps.Feedser.Source.Acsc/Class1.cs new file mode 100644 index 00000000..f03ee8c2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Acsc/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Acsc; + +public sealed class AcscConnectorPlugin : IConnectorPlugin +{ + public string Name => "acsc"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj b/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Cccs/Class1.cs b/src/StellaOps.Feedser.Source.Cccs/Class1.cs new file mode 100644 index 00000000..7274382e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cccs/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Cccs; + +public sealed class CccsConnectorPlugin : IConnectorPlugin +{ + public string Name => "cccs"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj b/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertBund/Class1.cs b/src/StellaOps.Feedser.Source.CertBund/Class1.cs new file mode 100644 index 00000000..358759b8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertBund/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertBund; + +public sealed class CertBundConnectorPlugin : IConnectorPlugin +{ + public string Name => "certbund"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj b/src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertCc/Class1.cs b/src/StellaOps.Feedser.Source.CertCc/Class1.cs new file mode 100644 index 00000000..48207316 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertCc; + +public sealed class CertCcConnectorPlugin : IConnectorPlugin +{ + public string Name => "certcc"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj b/src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs new file mode 100644 index 00000000..4323cc1d --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs @@ -0,0 +1,305 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Source.CertFr; +using StellaOps.Feedser.Source.CertFr.Configuration; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Source.CertFr.Tests; + +[Collection("mongo-fixture")] +public sealed class CertFrConnectorTests : IAsyncLifetime +{ + private static readonly Uri FeedUri = new("https://www.cert.ssi.gouv.fr/feed/alertes/"); + private static readonly Uri FirstDetailUri = new("https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/"); + private static readonly Uri SecondDetailUri = new("https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public CertFrConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 10, 3, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesDeterministicSnapshot() + { + await using var provider = await BuildServiceProviderAsync(); + SeedFeed(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var snapshot = SnapshotSerializer.ToSnapshot(advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray()); + var expected = ReadFixture("certfr-advisories.snapshot.json"); + var normalizedSnapshot = Normalize(snapshot); + var normalizedExpected = Normalize(expected); + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "CertFr", "Fixtures", "certfr-advisories.actual.json"); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + + var documentStore = provider.GetRequiredService(); + var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0); + } + + [Fact] + public async Task FetchFailure_RecordsBackoffAndReason() + { + await using var provider = await BuildServiceProviderAsync(); + _handler.AddResponse(FeedUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("feed error", Encoding.UTF8, "text/plain"), + }); + + var connector = provider.GetRequiredService(); + await Assert.ThrowsAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal(1, state!.FailCount); + Assert.NotNull(state.LastFailureReason); + Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal); + Assert.NotNull(state.BackoffUntil); + Assert.True(state.BackoffUntil > _timeProvider.GetUtcNow()); + } + + [Fact] + public async Task Fetch_NotModifiedResponsesMaintainDocumentState() + { + await using var provider = await BuildServiceProviderAsync(); + SeedFeed(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + SeedFeed(); + SeedNotModifiedDetailResponses(); + + await connector.FetchAsync(provider, CancellationToken.None); + + firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0); + } + + [Fact] + public async Task Fetch_DuplicateContentSkipsRequeue() + { + await using var provider = await BuildServiceProviderAsync(); + SeedFeed(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + var secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + SeedFeed(); + SeedDetailResponses(); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + firstDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, FirstDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal(DocumentStatuses.Mapped, firstDocument!.Status); + + secondDocument = await documentStore.FindBySourceAndUriAsync(CertFrConnectorPlugin.SourceName, SecondDetailUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal(DocumentStatuses.Mapped, secondDocument!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertFrConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMaps) && pendingMaps.AsBsonArray.Count == 0); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddCertFrConnector(opts => + { + opts.FeedUri = FeedUri; + opts.InitialBackfill = TimeSpan.FromDays(30); + opts.WindowOverlap = TimeSpan.FromDays(2); + opts.MaxItemsPerFetch = 50; + }); + + services.Configure(CertFrOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedFeed() + { + _handler.AddTextResponse(FeedUri, ReadFixture("certfr-feed.xml"), "application/atom+xml"); + } + + private void SeedDetailResponses() + { + AddDetailResponse(FirstDetailUri, "certfr-detail-AV-2024-001.html", "\"certfr-001\""); + AddDetailResponse(SecondDetailUri, "certfr-detail-AV-2024-002.html", "\"certfr-002\""); + } + + private void SeedNotModifiedDetailResponses() + { + AddNotModifiedResponse(FirstDetailUri, "\"certfr-001\""); + AddNotModifiedResponse(SecondDetailUri, "\"certfr-002\""); + } + + private void AddDetailResponse(Uri uri, string fixture, string? etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private void AddNotModifiedResponse(Uri uri, string? etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "CertFr", "Fixtures", filename); + return File.ReadAllText(path); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json new file mode 100644 index 00000000..ddc2efa8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json @@ -0,0 +1,112 @@ +[ + { + "advisoryKey": "cert-fr/AV-2024.001", + "affectedPackages": [], + "aliases": [ + "CERT-FR:AV-2024.001" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "fr", + "modified": null, + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + } + ], + "published": "2024-10-03T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + }, + "sourceTag": null, + "summary": null, + "url": "https://vendor.example.com/patch" + }, + { + "kind": "advisory", + "provenance": { + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + }, + "sourceTag": "cert-fr", + "summary": "Résumé de la première alerte.", + "url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + } + ], + "severity": null, + "summary": "Résumé de la première alerte.", + "title": "AV-2024.001 - Première alerte" + }, + { + "advisoryKey": "cert-fr/AV-2024.002", + "affectedPackages": [], + "aliases": [ + "CERT-FR:AV-2024.002" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "fr", + "modified": null, + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + } + ], + "published": "2024-10-03T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.example.com/kb/KB-1234" + }, + { + "kind": "reference", + "provenance": { + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.example.com/kb/KB-5678" + }, + { + "kind": "advisory", + "provenance": { + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": "cert-fr", + "summary": "Résumé de la deuxième alerte.", + "url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + } + ], + "severity": null, + "summary": "Résumé de la deuxième alerte.", + "title": "AV-2024.002 - Deuxième alerte" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html new file mode 100644 index 00000000..60cf7065 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html @@ -0,0 +1,8 @@ + + AV-2024.001 + +

    Alerte CERT-FR AV-2024.001

    +

    L'exploitation active de la vulnérabilité est surveillée.

    +

    Consultez les indications du fournisseur.

    + + diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html new file mode 100644 index 00000000..a3895ec0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html @@ -0,0 +1,11 @@ + + AV-2024.002 + +

    Alerte CERT-FR AV-2024.002

    +

    Des correctifs sont disponibles pour plusieurs produits.

    + + + diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml new file mode 100644 index 00000000..7ede5458 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml @@ -0,0 +1,22 @@ + + + + CERT-FR Alertes + https://www.cert.ssi.gouv.fr/ + Alertes example feed + + AV-2024.001 - Première alerte + https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/ + + Thu, 03 Oct 2024 09:00:00 +0000 + AV-2024.001 + + + AV-2024.002 - Deuxième alerte + https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/ + + Thu, 03 Oct 2024 11:30:00 +0000 + AV-2024.002 + + + diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj b/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj new file mode 100644 index 00000000..d2c7c787 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertFr/AGENTS.md b/src/StellaOps.Feedser.Source.CertFr/AGENTS.md new file mode 100644 index 00000000..5049da1f --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +ANSSI CERT-FR advisories connector (avis/alertes) providing national enrichment: advisory metadata, CVE links, mitigation notes, and references. +## Scope +- Harvest CERT-FR items via RSS and/or list pages; follow item pages for detail; window by publish/update date. +- Validate HTML or JSON payloads; extract structured fields; map to canonical aliases, references, severity text. +- Maintain watermarks and de-duplication by content hash; idempotent processing. +## Participants +- Source.Common (HTTP, HTML parsing helpers, validators). +- Storage.Mongo (document, dto, advisory, reference, source_state). +- Models (canonical). +- Core/WebService (jobs: source:certfr:fetch|parse|map). +- Merge engine (later) to enrich only. +## Interfaces & contracts +- Treat CERT-FR as enrichment; never override distro or PSIRT version ranges absent concrete evidence. +- References must include primary bulletin URL and vendor links; tag kind=bulletin/vendor/mitigation appropriately. +- Provenance records cite "cert-fr" with method=parser and source URL. +## In/Out of scope +In: advisory metadata extraction, references, severity text, watermarking. +Out: OVAL or package-level authority. +## Observability & security expectations +- Metrics: certfr.fetch.items, certfr.parse.fail, certfr.map.count. +- Logs: feed URL(s), item ids/urls, extraction durations; no PII; allowlist hostnames. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.CertFr.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs b/src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs new file mode 100644 index 00000000..662b16df --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs @@ -0,0 +1,337 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Source.CertFr.Configuration; +using StellaOps.Feedser.Source.CertFr.Internal; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertFr; + +public sealed class CertFrConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly CertFrFeedClient _feedClient; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CertFrOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public CertFrConnector( + CertFrFeedClient feedClient, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CertFrConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var windowEnd = now; + var lastPublished = cursor.LastPublished ?? now - _options.InitialBackfill; + var windowStart = lastPublished - _options.WindowOverlap; + var minStart = now - _options.InitialBackfill; + if (windowStart < minStart) + { + windowStart = minStart; + } + + IReadOnlyList items; + try + { + items = await _feedClient.LoadAsync(windowStart, windowEnd, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR feed load failed {Start:o}-{End:o}", windowStart, windowEnd); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (items.Count == 0) + { + await UpdateCursorAsync(cursor.WithLastPublished(windowEnd), cancellationToken).ConfigureAwait(false); + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailUri.ToString(), cancellationToken).ConfigureAwait(false); + var request = new SourceFetchRequest(CertFrOptions.HttpClientName, SourceName, item.DetailUri) + { + Metadata = CertFrDocumentMetadata.CreateMetadata(item), + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified || !result.IsSuccess || result.Document is null) + { + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + + continue; + } + + if (existing is not null + && string.Equals(existing.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase) + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + + continue; + } + + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR fetch failed for {Uri}", item.DetailUri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + if (maxPublished == DateTimeOffset.MinValue) + { + maxPublished = cursor.LastPublished ?? windowEnd; + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastPublished(maxPublished); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Cert-FR document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + CertFrDocumentMetadata metadata; + try + { + metadata = CertFrDocumentMetadata.FromDocument(document); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR metadata parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + CertFrDto dto; + try + { + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = System.Text.Encoding.UTF8.GetString(content); + dto = CertFrParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR parse failed for advisory {AdvisoryId} ({Uri})", metadata.AdvisoryId, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = BsonDocument.Parse(json); + var validatedAt = _timeProvider.GetUtcNow(); + + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certfr.detail.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "certfr.detail.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + CertFrDto? dto; + try + { + var json = dtoRecord.Payload.ToJson(); + dto = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cert-FR DTO deserialization failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("Cert-FR DTO payload deserialized as null for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var mappedAt = _timeProvider.GetUtcNow(); + var advisory = CertFrMapper.Map(dto, SourceName, mappedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return CertFrCursor.FromBson(record?.Cursor); + } + + private async Task UpdateCursorAsync(CertFrCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs b/src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs new file mode 100644 index 00000000..f53760ce --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertFr; + +public sealed class CertFrConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cert-fr"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs new file mode 100644 index 00000000..1abb07b4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.CertFr.Configuration; + +namespace StellaOps.Feedser.Source.CertFr; + +public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:cert-fr"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCertFrConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, CertFrJobKinds.Fetch, typeof(CertFrFetchJob)); + EnsureJob(options, CertFrJobKinds.Parse, typeof(CertFrParseJob)); + EnsureJob(options, CertFrJobKinds.Map, typeof(CertFrMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs new file mode 100644 index 00000000..0505f003 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs @@ -0,0 +1,36 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertFr.Configuration; +using StellaOps.Feedser.Source.CertFr.Internal; +using StellaOps.Feedser.Source.Common.Http; + +namespace StellaOps.Feedser.Source.CertFr; + +public static class CertFrServiceCollectionExtensions +{ + public static IServiceCollection AddCertFrConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(CertFrOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.FeedUri; + clientOptions.UserAgent = "StellaOps.Feedser.CertFr/1.0"; + clientOptions.Timeout = TimeSpan.FromSeconds(20); + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.FeedUri.Host); + }); + + services.TryAddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs b/src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs new file mode 100644 index 00000000..83599152 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs @@ -0,0 +1,46 @@ +using System; + +namespace StellaOps.Feedser.Source.CertFr.Configuration; + +public sealed class CertFrOptions +{ + public const string HttpClientName = "cert-fr"; + + public Uri FeedUri { get; set; } = new("https://www.cert.ssi.gouv.fr/feed/alertes/"); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2); + + public int MaxItemsPerFetch { get; set; } = 100; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.Zero; + + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Cert-FR FeedUri must be an absolute URI."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("InitialBackfill must be a positive duration."); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new InvalidOperationException("WindowOverlap cannot be negative."); + } + + if (MaxItemsPerFetch <= 0) + { + throw new InvalidOperationException("MaxItemsPerFetch must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs new file mode 100644 index 00000000..3f195864 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal sealed record CertFrCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static CertFrCursor Empty { get; } = new(null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + return document; + } + + public static CertFrCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPublished = document.TryGetValue("lastPublished", out var value) + ? ParseDate(value) + : null; + + return new CertFrCursor( + lastPublished, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings")); + } + + public CertFrCursor WithLastPublished(DateTimeOffset? timestamp) + => this with { LastPublished = timestamp }; + + public CertFrCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public CertFrCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var raw) || raw is not BsonArray array) + { + return Array.Empty(); + } + + var result = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + result.Add(guid); + } + } + + return result; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs new file mode 100644 index 00000000..c889d138 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal sealed record CertFrDocumentMetadata( + string AdvisoryId, + string Title, + DateTimeOffset Published, + Uri DetailUri, + string? Summary) +{ + private const string AdvisoryIdKey = "certfr.advisoryId"; + private const string TitleKey = "certfr.title"; + private const string PublishedKey = "certfr.published"; + private const string SummaryKey = "certfr.summary"; + + public static CertFrDocumentMetadata FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + if (document.Metadata is null) + { + throw new InvalidOperationException("Cert-FR document metadata is missing."); + } + + var metadata = document.Metadata; + if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("Cert-FR advisory id metadata missing."); + } + + if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title)) + { + throw new InvalidOperationException("Cert-FR title metadata missing."); + } + + if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published)) + { + throw new InvalidOperationException("Cert-FR published metadata invalid."); + } + + if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri)) + { + throw new InvalidOperationException("Cert-FR document URI invalid."); + } + + metadata.TryGetValue(SummaryKey, out var summary); + + return new CertFrDocumentMetadata( + advisoryId.Trim(), + title.Trim(), + published.ToUniversalTime(), + detailUri, + string.IsNullOrWhiteSpace(summary) ? null : summary.Trim()); + } + + public static IReadOnlyDictionary CreateMetadata(CertFrFeedItem item) + { + ArgumentNullException.ThrowIfNull(item); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + [AdvisoryIdKey] = item.AdvisoryId, + [TitleKey] = item.Title ?? item.AdvisoryId, + [PublishedKey] = item.Published.ToString("O"), + }; + + if (!string.IsNullOrWhiteSpace(item.Summary)) + { + metadata[SummaryKey] = item.Summary!; + } + + return metadata; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs new file mode 100644 index 00000000..2163642f --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal sealed record CertFrDto( + [property: JsonPropertyName("advisoryId")] string AdvisoryId, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("detailUrl")] string DetailUrl, + [property: JsonPropertyName("published")] DateTimeOffset Published, + [property: JsonPropertyName("summary")] string? Summary, + [property: JsonPropertyName("content")] string Content, + [property: JsonPropertyName("references")] IReadOnlyList References); diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs new file mode 100644 index 00000000..2126381c --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertFr.Configuration; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +public sealed class CertFrFeedClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly CertFrOptions _options; + private readonly ILogger _logger; + + public CertFrFeedClient(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(CertFrOptions.HttpClientName); + + using var response = await client.GetAsync(_options.FeedUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var document = XDocument.Load(stream); + + var items = new List(); + var now = DateTimeOffset.UtcNow; + + foreach (var itemElement in document.Descendants("item")) + { + var link = itemElement.Element("link")?.Value; + if (string.IsNullOrWhiteSpace(link) || !Uri.TryCreate(link.Trim(), UriKind.Absolute, out var detailUri)) + { + continue; + } + + var title = itemElement.Element("title")?.Value?.Trim(); + var summary = itemElement.Element("description")?.Value?.Trim(); + + var published = ParsePublished(itemElement.Element("pubDate")?.Value) ?? now; + if (published < windowStart) + { + continue; + } + + if (published > windowEnd) + { + published = windowEnd; + } + + var advisoryId = ResolveAdvisoryId(itemElement, detailUri); + items.Add(new CertFrFeedItem(advisoryId, detailUri, published.ToUniversalTime(), title, summary)); + } + + return items + .OrderBy(item => item.Published) + .Take(_options.MaxItemsPerFetch) + .ToArray(); + } + + private static DateTimeOffset? ParsePublished(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return parsed; + } + + return null; + } + + private static string ResolveAdvisoryId(XElement itemElement, Uri detailUri) + { + var guid = itemElement.Element("guid")?.Value; + if (!string.IsNullOrWhiteSpace(guid)) + { + return guid.Trim(); + } + + var segments = detailUri.Segments; + if (segments.Length > 0) + { + var slug = segments[^1].Trim('/'); + if (!string.IsNullOrWhiteSpace(slug)) + { + return slug; + } + } + + return detailUri.AbsoluteUri; + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs new file mode 100644 index 00000000..91a74ab7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs @@ -0,0 +1,10 @@ +using System; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +public sealed record CertFrFeedItem( + string AdvisoryId, + Uri DetailUri, + DateTimeOffset Published, + string? Title, + string? Summary); diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs new file mode 100644 index 00000000..d4779d23 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal static class CertFrMapper +{ + public static Advisory Map(CertFrDto dto, string sourceName, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + var advisoryKey = $"cert-fr/{dto.AdvisoryId}"; + var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime()); + + var aliases = new List + { + $"CERT-FR:{dto.AdvisoryId}", + }; + + var references = BuildReferences(dto, provenance).ToArray(); + + return new Advisory( + advisoryKey, + dto.Title, + dto.Summary ?? dto.Title, + language: "fr", + published: dto.Published.ToUniversalTime(), + modified: null, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static IEnumerable BuildReferences(CertFrDto dto, AdvisoryProvenance provenance) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var entries = new List<(AdvisoryReference Reference, int Priority)> + { + (new AdvisoryReference(dto.DetailUrl, "advisory", "cert-fr", dto.Summary, provenance), 0), + }; + + foreach (var url in dto.References) + { + entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1)); + } + + return entries + .GroupBy(tuple => tuple.Reference.Url, comparer) + .Select(group => group + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .First()) + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .Select(t => t.Reference); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs new file mode 100644 index 00000000..60fb81e4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.CertFr.Internal; + +internal static class CertFrParser +{ + private static readonly Regex AnchorRegex = new("]+href=\"(?https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex ScriptRegex = new("", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex StyleRegex = new("", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled); + private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled); + + public static CertFrDto Parse(string html, CertFrDocumentMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + var sanitized = SanitizeHtml(html); + var summary = BuildSummary(metadata.Summary, sanitized); + var references = ExtractReferences(html); + + return new CertFrDto( + metadata.AdvisoryId, + metadata.Title, + metadata.DetailUri.ToString(), + metadata.Published, + summary, + sanitized, + references); + } + + private static string SanitizeHtml(string html) + { + var withoutScripts = ScriptRegex.Replace(html, string.Empty); + var withoutStyles = StyleRegex.Replace(withoutScripts, string.Empty); + var withoutTags = TagRegex.Replace(withoutStyles, " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty; + return WhitespaceRegex.Replace(decoded, " ").Trim(); + } + + private static string? BuildSummary(string? metadataSummary, string content) + { + if (!string.IsNullOrWhiteSpace(metadataSummary)) + { + return metadataSummary.Trim(); + } + + if (string.IsNullOrWhiteSpace(content)) + { + return null; + } + + var sentences = content.Split(new[] { '.','!','?' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (sentences.Length > 0) + { + return sentences[0].Trim(); + } + + return content.Length > 280 ? content[..280].Trim() : content; + } + + private static IReadOnlyList ExtractReferences(string html) + { + var references = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in AnchorRegex.Matches(html)) + { + if (match.Success) + { + references.Add(match.Groups["url"].Value.Trim()); + } + } + + return references.Count == 0 + ? Array.Empty() + : references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertFr/Jobs.cs b/src/StellaOps.Feedser.Source.CertFr/Jobs.cs new file mode 100644 index 00000000..e7e1e601 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.CertFr; + +internal static class CertFrJobKinds +{ + public const string Fetch = "source:cert-fr:fetch"; + public const string Parse = "source:cert-fr:parse"; + public const string Map = "source:cert-fr:map"; +} + +internal sealed class CertFrFetchJob : IJob +{ + private readonly CertFrConnector _connector; + + public CertFrFetchJob(CertFrConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class CertFrParseJob : IJob +{ + private readonly CertFrConnector _connector; + + public CertFrParseJob(CertFrConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class CertFrMapJob : IJob +{ + private readonly CertFrConnector _connector; + + public CertFrMapJob(CertFrConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj b/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj new file mode 100644 index 00000000..9e3f378e --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertFr/TASKS.md b/src/StellaOps.Feedser.Source.CertFr/TASKS.md new file mode 100644 index 00000000..25f219ab --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertFr/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|RSS/list fetcher with sliding window|BE-Conn-CertFr|Source.Common|**DONE** – RSS/list ingestion implemented with sliding date cursor.| +|Detail page fetch and sanitizer|BE-Conn-CertFr|Source.Common|**DONE** – HTML sanitizer trims boilerplate prior to DTO mapping.| +|Extractor and schema validation of DTO|BE-Conn-CertFr, QA|Source.Common|**DONE** – DTO parsing validates structure before persistence.| +|Canonical mapping (aliases, refs, severity text)|BE-Conn-CertFr|Models|**DONE** – mapper emits enrichment references with severity text.| +|Watermark plus dedupe by sha256|BE-Conn-CertFr|Storage.Mongo|**DONE** – SHA comparisons skip unchanged docs; covered by duplicate/not-modified connector tests.| +|Golden fixtures and determinism tests|QA|Source.CertFr|**DONE** – snapshot fixtures added in `CertFrConnectorTests` to enforce deterministic output.| +|Mark failure/backoff on fetch errors|BE-Conn-CertFr|Storage.Mongo|**DONE** – fetch path now marks failures/backoff and tests assert state repository updates.| +|Conditional fetch caching|BE-Conn-CertFr|Source.Common|**DONE** – ETag/Last-Modified support wired via `SourceFetchService` and verified in not-modified test.| diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs new file mode 100644 index 00000000..af48a338 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs @@ -0,0 +1,340 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.CertIn; +using StellaOps.Feedser.Source.CertIn.Configuration; +using StellaOps.Feedser.Source.CertIn.Internal; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.CertIn.Tests; + +[Collection("mongo-fixture")] +public sealed class CertInConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public CertInConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 20, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_GeneratesExpectedSnapshot() + { + var options = new CertInOptions + { + AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(60), + WindowOverlap = TimeSpan.FromDays(7), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + _handler.Clear(); + + _handler.AddTextResponse(options.AlertsEndpoint, ReadFixture("alerts-page1.json"), "application/json"); + var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005"); + _handler.AddTextResponse(detailUri, ReadFixture("detail-CIAD-2024-0005.html"), "text/html"); + + var connector = new CertInConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + Assert.Single(advisories); + var canonical = SnapshotSerializer.ToSnapshot(advisories.Single()); + var expected = ReadFixture("expected-advisory.json"); + var normalizedExpected = NormalizeLineEndings(expected); + var normalizedActual = NormalizeLineEndings(canonical); + if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "CertIn", "Fixtures", "expected-advisory.actual.json"); + File.WriteAllText(actualPath, canonical); + } + + Assert.Equal(normalizedExpected, normalizedActual); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pending)); + Assert.Empty(pending.AsBsonArray); + } + + [Fact] + public async Task FetchFailure_RecordsBackoffAndReason() + { + var options = new CertInOptions + { + AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(60), + WindowOverlap = TimeSpan.FromDays(7), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + _handler.Clear(); + _handler.AddResponse(options.AlertsEndpoint, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("{}", Encoding.UTF8, "application/json"), + }); + + var provider = _serviceProvider!; + var connector = new CertInConnectorPlugin().Create(provider); + + await Assert.ThrowsAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal(1, state!.FailCount); + Assert.NotNull(state.LastFailureReason); + Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal); + Assert.True(state.BackoffUntil.HasValue); + Assert.True(state.BackoffUntil!.Value > _timeProvider.GetUtcNow()); + } + + [Fact] + public async Task Fetch_NotModifiedMaintainsDocumentState() + { + var options = new CertInOptions + { + AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(7), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + + var listingPayload = ReadFixture("alerts-page1.json"); + var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005"); + var detailHtml = ReadFixture("detail-CIAD-2024-0005.html"); + var etag = new EntityTagHeaderValue("\"certin-2024-0005\""); + var lastModified = new DateTimeOffset(2024, 4, 15, 10, 0, 0, TimeSpan.Zero); + + _handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json"); + _handler.AddResponse(detailUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(detailHtml, Encoding.UTF8, "text/html"), + }; + + response.Headers.ETag = etag; + response.Content.Headers.LastModified = lastModified; + return response; + }); + + var connector = new CertInConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + Assert.Equal(etag.Tag, document.Etag); + + _handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json"); + _handler.AddResponse(detailUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified) + { + Content = new StringContent(string.Empty) + }; + response.Headers.ETag = etag; + return response; + }); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Equal(0, pendingDocs.AsBsonArray.Count); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)); + Assert.Equal(0, pendingMappings.AsBsonArray.Count); + } + + [Fact] + public async Task Fetch_DuplicateContentSkipsRequeue() + { + var options = new CertInOptions + { + AlertsEndpoint = new Uri("https://cert-in.example/api/alerts", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(7), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + + var listingPayload = ReadFixture("alerts-page1.json"); + var detailUri = new Uri("https://cert-in.example/advisory/CIAD-2024-0005"); + var detailHtml = ReadFixture("detail-CIAD-2024-0005.html"); + + _handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json"); + _handler.AddTextResponse(detailUri, detailHtml, "text/html"); + + var connector = new CertInConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + _handler.AddTextResponse(options.AlertsEndpoint, listingPayload, "application/json"); + _handler.AddTextResponse(detailUri, detailHtml, "text/html"); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + document = await documentStore.FindBySourceAndUriAsync(CertInConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertInConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Equal(0, pendingDocs.AsBsonArray.Count); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)); + Assert.Equal(0, pendingMappings.AsBsonArray.Count); + } + + private async Task EnsureServiceProviderAsync(CertInOptions template) + { + if (_serviceProvider is not null) + { + await ResetDatabaseAsync(); + return; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddCertInConnector(opts => + { + opts.AlertsEndpoint = template.AlertsEndpoint; + opts.WindowSize = template.WindowSize; + opts.WindowOverlap = template.WindowOverlap; + opts.MaxPagesPerFetch = template.MaxPagesPerFetch; + opts.RequestDelay = template.RequestDelay; + }); + + services.Configure(CertInOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + _serviceProvider = services.BuildServiceProvider(); + var bootstrapper = _serviceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + } + + private Task ResetDatabaseAsync() + => _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "CertIn", "Fixtures", filename); + return File.ReadAllText(path); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider?.Dispose(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json new file mode 100644 index 00000000..a37d9230 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json @@ -0,0 +1,9 @@ +[ + { + "advisoryId": "CIAD-2024-0005", + "title": "Multiple vulnerabilities in Example Gateway", + "publishedOn": "2024-04-15T10:00:00Z", + "detailUrl": "https://cert-in.example/advisory/CIAD-2024-0005", + "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990)." + } +] diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html new file mode 100644 index 00000000..945bfb00 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html @@ -0,0 +1,17 @@ + + + + + Multiple vulnerabilities in Example Gateway + + +
    +

    Multiple vulnerabilities in Example Gateway

    +

    Severity: High

    +

    Vendor: Example Gateway Technologies Pvt Ltd

    +

    Organisation: Partner Systems Inc.

    +

    CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands.

    +

    Further information is available from the vendor bulletin.

    +
    + + diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json new file mode 100644 index 00000000..600f4fa1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json @@ -0,0 +1,97 @@ +{ + "advisoryKey": "CIAD-2024-0005", + "affectedPackages": [ + { + "identifier": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + } + ], + "aliases": [ + "CIAD-2024-0005", + "CVE-2024-9990", + "CVE-2024-9991" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-15T10:00:00+00:00", + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-04-20T00:00:00+00:00", + "source": "cert-in", + "value": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + { + "kind": "mapping", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "CIAD-2024-0005" + } + ], + "published": "2024-04-15T10:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + "sourceTag": "cert-in", + "summary": null, + "url": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + { + "kind": "reference", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://vendor.example.com/advisories/example-gateway-bulletin" + }, + "sourceTag": null, + "summary": null, + "url": "https://vendor.example.com/advisories/example-gateway-bulletin" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9990" + }, + "sourceTag": "CVE-2024-9990", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9990" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9991" + }, + "sourceTag": "CVE-2024-9991", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9991" + } + ], + "severity": "high vendor", + "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).", + "title": "Multiple vulnerabilities in Example Gateway" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj b/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj new file mode 100644 index 00000000..734d7e82 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertIn/AGENTS.md b/src/StellaOps.Feedser.Source.CertIn/AGENTS.md new file mode 100644 index 00000000..ebb7215e --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +CERT-In national CERT connector; enrichment advisories for India; maps CVE lists, advisory text, mitigations, and references; non-authoritative for package ranges unless explicit evidence is present. +## Scope +- Discover and fetch advisories from the CERT-In portal; window by advisory code/date; follow detail pages. +- Validate HTML or JSON; extract title, summary, CVEs, affected vendor names, mitigations; map references; normalize dates and IDs. +- Persist raw docs and maintain source_state cursor; idempotent mapping. +## Participants +- Source.Common (HTTP, HTML parsing, normalization, validators). +- Storage.Mongo (document, dto, advisory, alias, reference, source_state). +- Models (canonical). +- Core/WebService (jobs: source:certin:fetch|parse|map). +- Merge engine treats CERT-In as enrichment (no override of PSIRT or OVAL without concrete ranges). +## Interfaces & contracts +- Aliases: advisory code if stable (scheme "CERT-IN") and CVE ids; if code is not stable, store as reference only. +- References typed: bulletin/advisory/vendor/mitigation; deduped. +- Affected omitted unless CERT-In publishes explicit version or fix details. +- Provenance: method=parser; value=advisory code or URL; recordedAt. +## In/Out of scope +In: enrichment, aliasing where stable, references, mitigation text. +Out: package range authority; scraping behind auth walls. +## Observability & security expectations +- Metrics: certin.fetch.items, certin.parse.fail, certin.map.enriched_count. +- Logs: advisory codes, CVE counts per advisory, timing; allowlist host; redact personal data if present. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.CertIn.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs b/src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs new file mode 100644 index 00000000..cc2dac64 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs @@ -0,0 +1,440 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.CertIn.Configuration; +using StellaOps.Feedser.Source.CertIn.Internal; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertIn; + +public sealed class CertInConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General) + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly CertInClient _client; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CertInOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public CertInConnector( + CertInClient client, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CertInConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var windowStart = cursor.LastPublished.HasValue + ? cursor.LastPublished.Value - _options.WindowOverlap + : now - _options.WindowSize; + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + + for (var page = 1; page <= _options.MaxPagesPerFetch; page++) + { + IReadOnlyList listings; + try + { + listings = await _client.GetListingsAsync(page, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "CERT-In listings fetch failed for page {Page}", page); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + if (listings.Count == 0) + { + break; + } + + foreach (var listing in listings.OrderByDescending(static item => item.Published)) + { + if (listing.Published < windowStart) + { + page = _options.MaxPagesPerFetch + 1; + break; + } + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["certin.advisoryId"] = listing.AdvisoryId, + ["certin.title"] = listing.Title, + ["certin.link"] = listing.DetailUri.ToString(), + ["certin.published"] = listing.Published.ToString("O") + }; + + if (!string.IsNullOrWhiteSpace(listing.Summary)) + { + metadata["certin.summary"] = listing.Summary!; + } + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, listing.DetailUri.ToString(), cancellationToken).ConfigureAwait(false); + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync( + new SourceFetchRequest(CertInOptions.HttpClientName, SourceName, listing.DetailUri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "CERT-In fetch failed for {Uri}", listing.DetailUri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(3), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (existing is not null + && string.Equals(existing.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase) + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + continue; + } + + pendingDocuments.Add(result.Document.Id); + if (listing.Published > maxPublished) + { + maxPublished = listing.Published; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithLastPublished(maxPublished == DateTimeOffset.MinValue ? cursor.LastPublished : maxPublished); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("CERT-In document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + if (!TryDeserializeListing(document.Metadata, out var listing)) + { + _logger.LogWarning("CERT-In metadata missing for {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download raw CERT-In document {DocumentId}", document.Id); + throw; + } + + var dto = CertInDetailParser.Parse(listing, rawBytes); + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "certin.v1", payload, _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var dtoJson = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + CertInAdvisoryDto dto; + try + { + dto = JsonSerializer.Deserialize(dtoJson, SerializerOptions) + ?? throw new InvalidOperationException("Deserialized CERT-In DTO is null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize CERT-In DTO for {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = MapAdvisory(dto, document, dtoRecord); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private Advisory MapAdvisory(CertInAdvisoryDto dto, DocumentRecord document, DtoRecord dtoRecord) + { + var fetchProvenance = new AdvisoryProvenance(SourceName, "document", document.Uri, document.FetchedAt); + var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", dto.AdvisoryId, dtoRecord.ValidatedAt); + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId, + }; + foreach (var cve in dto.CveIds) + { + aliases.Add(cve); + } + + var references = new List(); + try + { + references.Add(new AdvisoryReference( + dto.Link, + "advisory", + "cert-in", + null, + new AdvisoryProvenance(SourceName, "reference", dto.Link, dtoRecord.ValidatedAt))); + } + catch (ArgumentException) + { + _logger.LogWarning("Invalid CERT-In link {Link} for advisory {AdvisoryId}", dto.Link, dto.AdvisoryId); + } + + foreach (var cve in dto.CveIds) + { + var url = $"https://www.cve.org/CVERecord?id={cve}"; + try + { + references.Add(new AdvisoryReference( + url, + "advisory", + cve, + null, + new AdvisoryProvenance(SourceName, "reference", url, dtoRecord.ValidatedAt))); + } + catch (ArgumentException) + { + // ignore invalid urls + } + } + + foreach (var link in dto.ReferenceLinks) + { + try + { + references.Add(new AdvisoryReference( + link, + "reference", + null, + null, + new AdvisoryProvenance(SourceName, "reference", link, dtoRecord.ValidatedAt))); + } + catch (ArgumentException) + { + // ignore invalid urls + } + } + + var affectedPackages = dto.VendorNames.Select(vendor => new AffectedPackage( + AffectedPackageTypes.IcsVendor, + vendor, + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance(SourceName, "affected", vendor, dtoRecord.ValidatedAt) + })) + .ToArray(); + + return new Advisory( + dto.AdvisoryId, + dto.Title, + dto.Summary ?? dto.Content, + language: "en", + published: dto.Published, + modified: dto.Published, + severity: dto.Severity, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mappingProvenance }); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? CertInCursor.Empty : CertInCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(CertInCursor cursor, CancellationToken cancellationToken) + { + return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken); + } + + private static bool TryDeserializeListing(IReadOnlyDictionary? metadata, out CertInListingItem listing) + { + listing = null!; + if (metadata is null) + { + return false; + } + + if (!metadata.TryGetValue("certin.advisoryId", out var advisoryId)) + { + return false; + } + + if (!metadata.TryGetValue("certin.title", out var title)) + { + return false; + } + + if (!metadata.TryGetValue("certin.link", out var link) || !Uri.TryCreate(link, UriKind.Absolute, out var detailUri)) + { + return false; + } + + if (!metadata.TryGetValue("certin.published", out var publishedText) || !DateTimeOffset.TryParse(publishedText, out var published)) + { + return false; + } + + metadata.TryGetValue("certin.summary", out var summary); + + listing = new CertInListingItem(advisoryId, title, detailUri, published.ToUniversalTime(), summary); + return true; + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs b/src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs new file mode 100644 index 00000000..eeec9485 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.CertIn; + +public sealed class CertInConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cert-in"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs new file mode 100644 index 00000000..a3476f13 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.CertIn.Configuration; + +namespace StellaOps.Feedser.Source.CertIn; + +public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:cert-in"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCertInConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, CertInJobKinds.Fetch, typeof(CertInFetchJob)); + EnsureJob(options, CertInJobKinds.Parse, typeof(CertInParseJob)); + EnsureJob(options, CertInJobKinds.Map, typeof(CertInMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs new file mode 100644 index 00000000..9128caad --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertIn.Configuration; +using StellaOps.Feedser.Source.CertIn.Internal; +using StellaOps.Feedser.Source.Common.Http; + +namespace StellaOps.Feedser.Source.CertIn; + +public static class CertInServiceCollectionExtensions +{ + public static IServiceCollection AddCertInConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(CertInOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.AlertsEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.CertIn/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddTransient(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs b/src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs new file mode 100644 index 00000000..7beede2e --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs @@ -0,0 +1,68 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.CertIn.Configuration; + +public sealed class CertInOptions +{ + public static string HttpClientName => "source.certin"; + + /// + /// Endpoint returning a paginated list of recent advisories. + /// + public Uri AlertsEndpoint { get; set; } = new("https://www.cert-in.org.in/api/alerts", UriKind.Absolute); + + /// + /// Size of the rolling fetch window. + /// + public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30); + + /// + /// Overlap applied to subsequent windows. + /// + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2); + + /// + /// Maximum pages fetched per cycle. + /// + public int MaxPagesPerFetch { get; set; } = 5; + + /// + /// Delay between successive HTTP requests. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500); + + [MemberNotNull(nameof(AlertsEndpoint))] + public void Validate() + { + if (AlertsEndpoint is null || !AlertsEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("AlertsEndpoint must be an absolute URI."); + } + + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("WindowSize must be greater than zero."); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new InvalidOperationException("WindowOverlap cannot be negative."); + } + + if (WindowOverlap >= WindowSize) + { + throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxPagesPerFetch must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs new file mode 100644 index 00000000..0ee1076c --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +internal sealed record CertInAdvisoryDto( + string AdvisoryId, + string Title, + string Link, + DateTimeOffset Published, + string? Summary, + string Content, + string? Severity, + ImmutableArray CveIds, + ImmutableArray VendorNames, + ImmutableArray ReferenceLinks); diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs new file mode 100644 index 00000000..c01fce0f --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.CertIn.Configuration; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +public sealed class CertInClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly CertInOptions _options; + private readonly ILogger _logger; + + public CertInClient(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> GetListingsAsync(int page, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(CertInOptions.HttpClientName); + var requestUri = BuildPageUri(_options.AlertsEndpoint, page); + + using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + + var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Array) + { + _logger.LogWarning("Unexpected CERT-In alert payload shape for {Uri}", requestUri); + return Array.Empty(); + } + + var items = new List(capacity: root.GetArrayLength()); + foreach (var element in root.EnumerateArray()) + { + if (!TryParseListing(element, out var item)) + { + continue; + } + + items.Add(item); + } + + return items; + } + + private static bool TryParseListing(JsonElement element, out CertInListingItem item) + { + item = null!; + + if (!element.TryGetProperty("advisoryId", out var idElement) || idElement.ValueKind != JsonValueKind.String) + { + return false; + } + + var advisoryId = idElement.GetString(); + if (string.IsNullOrWhiteSpace(advisoryId)) + { + return false; + } + + var title = element.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String + ? titleElement.GetString() + : advisoryId; + + if (!element.TryGetProperty("detailUrl", out var linkElement) || linkElement.ValueKind != JsonValueKind.String) + { + return false; + } + + if (!Uri.TryCreate(linkElement.GetString(), UriKind.Absolute, out var detailUri)) + { + return false; + } + + DateTimeOffset published; + if (element.TryGetProperty("publishedOn", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String) + { + if (!DateTimeOffset.TryParse(publishedElement.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out published)) + { + return false; + } + } + else + { + return false; + } + + string? summary = null; + if (element.TryGetProperty("summary", out var summaryElement) && summaryElement.ValueKind == JsonValueKind.String) + { + summary = summaryElement.GetString(); + } + + item = new CertInListingItem(advisoryId.Trim(), title?.Trim() ?? advisoryId.Trim(), detailUri, published.ToUniversalTime(), summary?.Trim()); + return true; + } + + private static Uri BuildPageUri(Uri baseUri, int page) + { + if (page <= 1) + { + return baseUri; + } + + var builder = new UriBuilder(baseUri); + var trimmed = builder.Query.TrimStart('?'); + var pageSegment = $"page={page.ToString(CultureInfo.InvariantCulture)}"; + builder.Query = string.IsNullOrEmpty(trimmed) + ? pageSegment + : $"{trimmed}&{pageSegment}"; + return builder.Uri; + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs new file mode 100644 index 00000000..227ca5b4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +internal sealed record CertInCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static CertInCursor Empty { get; } = new(null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + return document; + } + + public static CertInCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPublished = document.TryGetValue("lastPublished", out var dateValue) + ? ParseDate(dateValue) + : null; + + return new CertInCursor( + lastPublished, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings")); + } + + public CertInCursor WithLastPublished(DateTimeOffset? timestamp) + => this with { LastPublished = timestamp }; + + public CertInCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public CertInCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs new file mode 100644 index 00000000..1154c298 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +internal static class CertInDetailParser +{ + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex SeverityRegex = new("Severity\\s*[:\\-]\\s*(?[A-Za-z ]{1,32})", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex VendorRegex = new("(?:Vendor|Organisation|Organization|Company)\\s*[:\\-]\\s*(?[^\\n\\r]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex LinkRegex = new("href=\"(https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static CertInAdvisoryDto Parse(CertInListingItem listing, byte[] rawHtml) + { + ArgumentNullException.ThrowIfNull(listing); + + var html = Encoding.UTF8.GetString(rawHtml); + var content = HtmlToPlainText(html); + var summary = listing.Summary ?? ExtractSummary(content); + var severity = ExtractSeverity(content); + var cves = ExtractCves(listing.Title, summary, content); + var vendors = ExtractVendors(summary, content); + var references = ExtractLinks(html); + + return new CertInAdvisoryDto( + listing.AdvisoryId, + listing.Title, + listing.DetailUri.ToString(), + listing.Published, + summary, + content, + severity, + cves, + vendors, + references); + } + + private static string HtmlToPlainText(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + var withoutScripts = Regex.Replace(html, "", string.Empty, RegexOptions.IgnoreCase); + var withoutStyles = Regex.Replace(withoutScripts, "", string.Empty, RegexOptions.IgnoreCase); + var withoutComments = Regex.Replace(withoutStyles, "", string.Empty, RegexOptions.Singleline); + var withoutTags = Regex.Replace(withoutComments, "<[^>]+>", " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags); + return string.IsNullOrWhiteSpace(decoded) + ? string.Empty + : Regex.Replace(decoded, "\\s+", " ").Trim(); + } + + private static string? ExtractSummary(string content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return null; + } + + var sentenceTerminators = new[] { ".", "!", "?" }; + foreach (var terminator in sentenceTerminators) + { + var index = content.IndexOf(terminator, StringComparison.Ordinal); + if (index > 0) + { + return content[..(index + terminator.Length)].Trim(); + } + } + + return content.Length > 280 ? content[..280].Trim() : content; + } + + private static string? ExtractSeverity(string content) + { + var match = SeverityRegex.Match(content); + if (match.Success) + { + return match.Groups["value"].Value.Trim().ToLowerInvariant(); + } + + return null; + } + + private static ImmutableArray ExtractCves(string title, string? summary, string content) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + + void Capture(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + foreach (Match match in CveRegex.Matches(text)) + { + if (match.Success) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + } + + Capture(title); + Capture(summary); + Capture(content); + + return set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); + } + + private static ImmutableArray ExtractVendors(string? summary, string content) + { + var vendors = new HashSet(StringComparer.OrdinalIgnoreCase); + + void Add(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + var cleaned = value + .Replace("’", "'", StringComparison.Ordinal) + .Trim(); + + if (cleaned.Length > 200) + { + cleaned = cleaned[..200]; + } + + if (!string.IsNullOrWhiteSpace(cleaned)) + { + vendors.Add(cleaned); + } + } + + if (!string.IsNullOrWhiteSpace(summary)) + { + foreach (Match match in VendorRegex.Matches(summary)) + { + Add(match.Groups["value"].Value); + } + } + + foreach (Match match in VendorRegex.Matches(content)) + { + Add(match.Groups["value"].Value); + } + + if (vendors.Count == 0 && !string.IsNullOrWhiteSpace(summary)) + { + var fallback = summary.Split('.', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + Add(fallback); + } + + return vendors.Count == 0 + ? ImmutableArray.Empty + : vendors.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); + } + + private static ImmutableArray ExtractLinks(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return ImmutableArray.Empty; + } + + var links = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in LinkRegex.Matches(html)) + { + if (match.Success) + { + links.Add(match.Groups[1].Value); + } + } + + return links.Count == 0 + ? ImmutableArray.Empty + : links.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs new file mode 100644 index 00000000..bc9f21dc --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs @@ -0,0 +1,10 @@ +using System; + +namespace StellaOps.Feedser.Source.CertIn.Internal; + +public sealed record CertInListingItem( + string AdvisoryId, + string Title, + Uri DetailUri, + DateTimeOffset Published, + string? Summary); diff --git a/src/StellaOps.Feedser.Source.CertIn/Jobs.cs b/src/StellaOps.Feedser.Source.CertIn/Jobs.cs new file mode 100644 index 00000000..95224193 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.CertIn; + +internal static class CertInJobKinds +{ + public const string Fetch = "source:cert-in:fetch"; + public const string Parse = "source:cert-in:parse"; + public const string Map = "source:cert-in:map"; +} + +internal sealed class CertInFetchJob : IJob +{ + private readonly CertInConnector _connector; + + public CertInFetchJob(CertInConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class CertInParseJob : IJob +{ + private readonly CertInConnector _connector; + + public CertInParseJob(CertInConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class CertInMapJob : IJob +{ + private readonly CertInConnector _connector; + + public CertInMapJob(CertInConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj b/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj new file mode 100644 index 00000000..07f798f6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.CertIn/TASKS.md b/src/StellaOps.Feedser.Source.CertIn/TASKS.md new file mode 100644 index 00000000..f25979ee --- /dev/null +++ b/src/StellaOps.Feedser.Source.CertIn/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Index/detail crawler with windowing|BE-Conn-CertIn|Source.Common|**DONE** – index/detail fetch implemented with window overlap and pagination.| +|Extractor (title/CVEs/mitigation)|BE-Conn-CertIn|Source.Common|**DONE** – parser normalizes encodings, CVE lists, and mitigation text.| +|DTO validation and sanitizer|BE-Conn-CertIn, QA|Source.Common|**DONE** – HTML sanitizer produces DTO before persistence.| +|Canonical mapping (aliases, refs)|BE-Conn-CertIn|Models|**DONE** – mapper creates CERT-IN aliases plus typed references.| +|State/dedupe and fixtures|BE-Conn-CertIn, QA|Storage.Mongo|**DONE** – snapshot/resume tests cover dedupe and cursor handling.| +|Mark failure/backoff on fetch errors|BE-Conn-CertIn|Storage.Mongo|**DONE** – fetch pipeline marks failures/backoff with unit coverage.| +|Conditional fetch caching|BE-Conn-CertIn|Source.Common|**DONE** – connector reuses ETag/Last-Modified; tests verify not-modified flow.| diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs new file mode 100644 index 00000000..21a9751b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs @@ -0,0 +1,37 @@ +using System.Net; +using System.Net.Http; +using StellaOps.Feedser.Source.Common.Testing; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class CannedHttpMessageHandlerTests +{ + [Fact] + public async Task SendAsync_RecordsRequestsAndSupportsFallback() + { + var handler = new CannedHttpMessageHandler(); + var requestUri = new Uri("https://example.test/api/resource"); + handler.AddResponse(HttpMethod.Get, requestUri, () => new HttpResponseMessage(HttpStatusCode.OK)); + handler.SetFallback(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + + using var client = handler.CreateClient(); + var firstResponse = await client.GetAsync(requestUri); + var secondResponse = await client.GetAsync(new Uri("https://example.test/other")); + + Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); + Assert.Equal(HttpStatusCode.NotFound, secondResponse.StatusCode); + Assert.Equal(2, handler.Requests.Count); + handler.AssertNoPendingResponses(); + } + + [Fact] + public async Task AddException_ThrowsDuringSend() + { + var handler = new CannedHttpMessageHandler(); + var requestUri = new Uri("https://example.test/api/error"); + handler.AddException(HttpMethod.Get, requestUri, new InvalidOperationException("boom")); + + using var client = handler.CreateClient(); + await Assert.ThrowsAsync(() => client.GetAsync(requestUri)); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs new file mode 100644 index 00000000..25320224 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs @@ -0,0 +1,31 @@ +using StellaOps.Feedser.Source.Common.Html; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class HtmlContentSanitizerTests +{ + [Fact] + public void Sanitize_RemovesScriptAndDangerousAttributes() + { + var sanitizer = new HtmlContentSanitizer(); + var input = "
    link
    "; + + var sanitized = sanitizer.Sanitize(input, new Uri("https://example.test/base/")); + + Assert.DoesNotContain("script", sanitized, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("onclick", sanitized, StringComparison.OrdinalIgnoreCase); + Assert.Contains("https://example.test/foo", sanitized, StringComparison.Ordinal); + Assert.Contains("rel=\"noopener nofollow noreferrer\"", sanitized, StringComparison.Ordinal); + } + + [Fact] + public void Sanitize_PreservesBasicFormatting() + { + var sanitizer = new HtmlContentSanitizer(); + var input = "

    Hello world

    "; + + var sanitized = sanitizer.Sanitize(input); + + Assert.Equal("

    Hello world

    ", sanitized); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs new file mode 100644 index 00000000..2a3f9e42 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs @@ -0,0 +1,41 @@ +using NuGet.Versioning; +using StellaOps.Feedser.Source.Common.Packages; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class PackageCoordinateHelperTests +{ + [Fact] + public void TryParsePackageUrl_ReturnsCanonicalForm() + { + var success = PackageCoordinateHelper.TryParsePackageUrl("pkg:npm/@scope/example@1.0.0?env=prod", out var coordinates); + + Assert.True(success); + Assert.NotNull(coordinates); + Assert.Equal("pkg:npm/@scope/example@1.0.0?env=prod", coordinates!.Canonical); + Assert.Equal("npm", coordinates.Type); + Assert.Equal("example", coordinates.Name); + Assert.Equal("1.0.0", coordinates.Version); + Assert.Equal("prod", coordinates.Qualifiers["env"]); + } + + [Fact] + public void TryParseSemVer_NormalizesVersion() + { + var success = PackageCoordinateHelper.TryParseSemVer("1.2.3+build", out var version, out var normalized); + + Assert.True(success); + Assert.Equal(SemanticVersion.Parse("1.2.3"), version); + Assert.Equal("1.2.3", normalized); + } + + [Fact] + public void TryParseSemVerRange_SupportsCaret() + { + var success = PackageCoordinateHelper.TryParseSemVerRange("^1.2.3", out var range); + + Assert.True(success); + Assert.NotNull(range); + Assert.True(range!.Satisfies(NuGetVersion.Parse("1.3.0"))); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs new file mode 100644 index 00000000..692eed57 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs @@ -0,0 +1,21 @@ +using StellaOps.Feedser.Source.Common.Pdf; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class PdfTextExtractorTests +{ + private const string SamplePdfBase64 = "JVBERi0xLjEKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA0NCA+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjcyIDcyMCBUZAooSGVsbG8gV29ybGQpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKNSAwIG9iago8PCAvVHlwZSAvRm9udCAvU3VidHlwZSAvVHlwZTEgL0Jhc2VGb250IC9IZWx2ZXRpY2EgPj4KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMTAgMDAwMDAgbiAKMDAwMDAwMDU2IDAwMDAwIG4gCjAwMDAwMDAxMTMgMDAwMDAgbiAKMDAwMDAwMDIxMCAwMDAwMCBuIAowMDAwMDAwMzExIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1Jvb3QgMSAwIFIgL1NpemUgNiA+PgpzdGFydHhyZWYKMzc3CiUlRU9G"; + + [Fact] + public async Task ExtractTextAsync_ReturnsPageText() + { + var bytes = Convert.FromBase64String(SamplePdfBase64); + using var stream = new MemoryStream(bytes); + var extractor = new PdfTextExtractor(); + + var result = await extractor.ExtractTextAsync(stream, cancellationToken: CancellationToken.None); + + Assert.Contains("Hello World", result.Text); + Assert.Equal(1, result.PagesProcessed); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs new file mode 100644 index 00000000..ec5788bb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs @@ -0,0 +1,36 @@ +using StellaOps.Feedser.Source.Common.Fetch; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class SourceFetchServiceTests +{ + [Fact] + public void CreateHttpRequestMessage_DefaultsToJsonAccept() + { + var request = new SourceFetchRequest("client", "source", new Uri("https://example.test/data")); + + using var message = SourceFetchService.CreateHttpRequestMessage(request); + + Assert.Single(message.Headers.Accept); + Assert.Equal("application/json", message.Headers.Accept.First().MediaType); + } + + [Fact] + public void CreateHttpRequestMessage_UsesAcceptOverrides() + { + var request = new SourceFetchRequest("client", "source", new Uri("https://example.test/data")) + { + AcceptHeaders = new[] + { + "text/html", + "application/xhtml+xml;q=0.9", + } + }; + + using var message = SourceFetchService.CreateHttpRequestMessage(request); + + Assert.Equal(2, message.Headers.Accept.Count); + Assert.Contains(message.Headers.Accept, h => h.MediaType == "text/html"); + Assert.Contains(message.Headers.Accept, h => h.MediaType == "application/xhtml+xml"); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs new file mode 100644 index 00000000..f404e1ea --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs @@ -0,0 +1,87 @@ +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class TimeWindowCursorPlannerTests +{ + [Fact] + public void GetNextWindow_UsesInitialBackfillWhenStateEmpty() + { + var now = new DateTimeOffset(2024, 10, 1, 12, 0, 0, TimeSpan.Zero); + var options = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromHours(4), + Overlap = TimeSpan.FromMinutes(15), + InitialBackfill = TimeSpan.FromDays(2), + MinimumWindowSize = TimeSpan.FromMinutes(1), + }; + + var window = TimeWindowCursorPlanner.GetNextWindow(now, null, options); + + Assert.Equal(now - options.InitialBackfill, window.Start); + Assert.Equal(window.Start + options.WindowSize, window.End); + } + + [Fact] + public void GetNextWindow_ClampsEndToNowWhenWindowExtendPastPresent() + { + var now = new DateTimeOffset(2024, 10, 10, 0, 0, 0, TimeSpan.Zero); + var options = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromHours(6), + Overlap = TimeSpan.FromMinutes(30), + InitialBackfill = TimeSpan.FromDays(3), + MinimumWindowSize = TimeSpan.FromMinutes(1), + }; + + var previousEnd = now - TimeSpan.FromMinutes(10); + var state = new TimeWindowCursorState(previousEnd - options.WindowSize, previousEnd); + + var window = TimeWindowCursorPlanner.GetNextWindow(now, state, options); + + var expectedStart = previousEnd - options.Overlap; + var earliest = now - options.InitialBackfill; + if (expectedStart < earliest) + { + expectedStart = earliest; + } + + Assert.Equal(expectedStart, window.Start); + Assert.Equal(now, window.End); + } + + [Fact] + public void TimeWindowCursorState_RoundTripThroughBson() + { + var state = new TimeWindowCursorState( + new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero), + new DateTimeOffset(2024, 9, 1, 6, 0, 0, TimeSpan.Zero)); + + var document = new BsonDocument + { + ["preserve"] = "value", + }; + + state.WriteTo(document); + var roundTripped = TimeWindowCursorState.FromBsonDocument(document); + + Assert.Equal(state.LastWindowStart, roundTripped.LastWindowStart); + Assert.Equal(state.LastWindowEnd, roundTripped.LastWindowEnd); + Assert.Equal("value", document["preserve"].AsString); + } + + [Fact] + public void PaginationPlanner_EnumeratesAdditionalPages() + { + var indices = PaginationPlanner.EnumerateAdditionalPages(4500, 2000).ToArray(); + Assert.Equal(new[] { 2000, 4000 }, indices); + } + + [Fact] + public void PaginationPlanner_ReturnsEmptyWhenSinglePage() + { + var indices = PaginationPlanner.EnumerateAdditionalPages(1000, 2000).ToArray(); + Assert.Empty(indices); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs new file mode 100644 index 00000000..c7881c04 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs @@ -0,0 +1,24 @@ +using StellaOps.Feedser.Source.Common.Url; + +namespace StellaOps.Feedser.Source.Common.Tests; + +public sealed class UrlNormalizerTests +{ + [Fact] + public void TryNormalize_ResolvesRelative() + { + var success = UrlNormalizer.TryNormalize("/foo/bar", new Uri("https://example.test/base/"), out var normalized); + + Assert.True(success); + Assert.Equal("https://example.test/foo/bar", normalized!.ToString()); + } + + [Fact] + public void TryNormalize_StripsFragment() + { + var success = UrlNormalizer.TryNormalize("https://example.test/path#section", null, out var normalized); + + Assert.True(success); + Assert.Equal("https://example.test/path", normalized!.ToString()); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs new file mode 100644 index 00000000..753f7bd2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs @@ -0,0 +1,51 @@ +using System; +using System.Text.Json; +using Json.Schema; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Source.Common.Json; + +namespace StellaOps.Feedser.Source.Common.Tests.Json; + +public sealed class JsonSchemaValidatorTests +{ + private static JsonSchema CreateSchema() + => JsonSchema.FromText(""" + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "count": { "type": "integer", "minimum": 1 } + }, + "required": ["id", "count"], + "additionalProperties": false + } + """); + + [Fact] + public void Validate_AllowsDocumentsMatchingSchema() + { + var schema = CreateSchema(); + using var document = JsonDocument.Parse("""{"id":"abc","count":2}"""); + var validator = new JsonSchemaValidator(NullLogger.Instance); + + var exception = Record.Exception(() => validator.Validate(document, schema, "valid-doc")); + + Assert.Null(exception); + } + + [Fact] + public void Validate_ThrowsWithDetailedViolations() + { + var schema = CreateSchema(); + using var document = JsonDocument.Parse("""{"count":0,"extra":"nope"}"""); + var validator = new JsonSchemaValidator(NullLogger.Instance); + + var ex = Assert.Throws(() => validator.Validate(document, schema, "invalid-doc")); + + Assert.Equal("invalid-doc", ex.DocumentName); + Assert.NotEmpty(ex.Errors); + Assert.Contains(ex.Errors, error => error.Keyword == "required"); + Assert.Contains(ex.Errors, error => error.SchemaLocation.Contains("#/additionalProperties", StringComparison.Ordinal)); + Assert.Contains(ex.Errors, error => error.Keyword == "minimum"); + } +} diff --git a/src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj b/src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj new file mode 100644 index 00000000..2e226f16 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + + + + + diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs b/src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs new file mode 100644 index 00000000..2825e7d0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs @@ -0,0 +1,58 @@ +using System.IO; +using System.Xml; +using System.Xml.Linq; +using System.Xml.Schema; +using Microsoft.Extensions.Logging.Abstractions; +using FeedserXmlSchemaValidator = StellaOps.Feedser.Source.Common.Xml.XmlSchemaValidator; +using FeedserXmlSchemaValidationException = StellaOps.Feedser.Source.Common.Xml.XmlSchemaValidationException; + +namespace StellaOps.Feedser.Source.Common.Tests.Xml; + +public sealed class XmlSchemaValidatorTests +{ + private static XmlSchemaSet CreateSchema() + { + var set = new XmlSchemaSet(); + set.Add(string.Empty, XmlReader.Create(new StringReader(""" + + + + + + + + + + + """))); + set.CompilationSettings = new XmlSchemaCompilationSettings { EnableUpaCheck = true }; + set.Compile(); + return set; + } + + [Fact] + public void Validate_AllowsCompliantDocument() + { + var schemaSet = CreateSchema(); + var document = XDocument.Parse("abc3"); + var validator = new FeedserXmlSchemaValidator(NullLogger.Instance); + + var exception = Record.Exception(() => validator.Validate(document, schemaSet, "valid.xml")); + + Assert.Null(exception); + } + + [Fact] + public void Validate_ThrowsWithDetailedErrors() + { + var schemaSet = CreateSchema(); + var document = XDocument.Parse("missing-count"); + var validator = new FeedserXmlSchemaValidator(NullLogger.Instance); + + var ex = Assert.Throws(() => validator.Validate(document, schemaSet, "invalid.xml")); + + Assert.Equal("invalid.xml", ex.DocumentName); + Assert.NotEmpty(ex.Errors); + Assert.Contains(ex.Errors, error => error.Message.Contains("count", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/AGENTS.md b/src/StellaOps.Feedser.Source.Common/AGENTS.md new file mode 100644 index 00000000..723af148 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/AGENTS.md @@ -0,0 +1,32 @@ +# AGENTS +## Role +Shared connector toolkit. Provides HTTP clients, retry/backoff, conditional GET (ETag/Last-Modified), schema validation, pagination helpers, clocks, and common DTO utilities for all connectors. +## Scope +- Typed HttpClient registrations with allowlisted hosts and timeouts. +- Request pipeline: retries with jitter, backoff on 429/5xx, rate-limit tracking per source. +- Conditional GET helpers (If-None-Match, If-Modified-Since), window cursors, and pagination iterators. +- Validators: JSON Schema, XML Schema (for example XmlSchemaValidator), and sanitizers. +- Content hashing and raw document capture helpers; metadata extraction (headers, status). +- HTML sanitization, URL normalization, and PDF-to-text extraction utilities for feeds that require cleanup before validation. +## Participants +- Source.* connectors (NVD, Red Hat, JVN, PSIRTs, CERTs, ICS). +- Storage.Mongo (document/dto repositories using shared shapes). +- Core (jobs schedule/trigger for connectors). +- QA (canned HTTP server harness, schema fixtures). +## Interfaces & contracts +- All network calls must pass through configured HttpClient with allowlist and sane timeouts; no direct new HttpClient(). +- Validators return detailed errors; invalid payloads quarantined and not mapped. +- Cursor helpers implement sliding windows and ID-based pagination; rely on IClock/TimeProvider for determinism. +- Strict provenance tags for extraction method: parser, oval, package.nevra, llm (gated). +## In/Out of scope +In: HTTP plumbing, validators, cursor/backoff utilities, hashing. +Out: connector-specific schemas/mapping rules, merge precedence. +## Observability & security expectations +- Metrics: http.req.count, http.retry.count, rate_limit.remaining, validator.fail.count. +- Logs include uri, status, retries, etag; redact tokens and auth headers. +- Distributed tracing hooks and per-connector counters should be wired centrally for consistent observability. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Common.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs b/src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs new file mode 100644 index 00000000..0d26babc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs @@ -0,0 +1,29 @@ +namespace StellaOps.Feedser.Source.Common.Cursors; + +/// +/// Provides helpers for computing pagination start indices for sources that expose total result counts. +/// +public static class PaginationPlanner +{ + /// + /// Enumerates additional page start indices given the total result count returned by the source. + /// The first page (at ) is assumed to be already fetched. + /// + public static IEnumerable EnumerateAdditionalPages(int totalResults, int resultsPerPage, int firstPageStartIndex = 0) + { + if (totalResults <= 0 || resultsPerPage <= 0) + { + yield break; + } + + if (firstPageStartIndex < 0) + { + firstPageStartIndex = 0; + } + + for (var start = firstPageStartIndex + resultsPerPage; start < totalResults; start += resultsPerPage) + { + yield return start; + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs new file mode 100644 index 00000000..6d127ca2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs @@ -0,0 +1,43 @@ +namespace StellaOps.Feedser.Source.Common.Cursors; + +/// +/// Configuration applied when advancing sliding time-window cursors. +/// +public sealed class TimeWindowCursorOptions +{ + public TimeSpan WindowSize { get; init; } = TimeSpan.FromHours(4); + + public TimeSpan Overlap { get; init; } = TimeSpan.FromMinutes(5); + + public TimeSpan InitialBackfill { get; init; } = TimeSpan.FromDays(7); + + public TimeSpan MinimumWindowSize { get; init; } = TimeSpan.FromMinutes(1); + + public void EnsureValid() + { + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("Window size must be positive."); + } + + if (Overlap < TimeSpan.Zero) + { + throw new InvalidOperationException("Window overlap cannot be negative."); + } + + if (Overlap >= WindowSize) + { + throw new InvalidOperationException("Window overlap must be less than the window size."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Initial backfill must be positive."); + } + + if (MinimumWindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("Minimum window size must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs new file mode 100644 index 00000000..c050d7ac --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs @@ -0,0 +1,50 @@ +namespace StellaOps.Feedser.Source.Common.Cursors; + +/// +/// Utility methods for computing sliding time-window ranges used by connectors. +/// +public static class TimeWindowCursorPlanner +{ + public static TimeWindow GetNextWindow(DateTimeOffset now, TimeWindowCursorState? state, TimeWindowCursorOptions options) + { + ArgumentNullException.ThrowIfNull(options); + options.EnsureValid(); + + var effectiveState = state ?? TimeWindowCursorState.Empty; + + var earliest = now - options.InitialBackfill; + var anchorEnd = effectiveState.LastWindowEnd ?? earliest; + if (anchorEnd < earliest) + { + anchorEnd = earliest; + } + + var start = anchorEnd - options.Overlap; + if (start < earliest) + { + start = earliest; + } + + var end = start + options.WindowSize; + if (end > now) + { + end = now; + } + + if (end <= start) + { + end = start + options.MinimumWindowSize; + if (end > now) + { + end = now; + } + } + + if (end <= start) + { + throw new InvalidOperationException("Unable to compute a non-empty time window with the provided options."); + } + + return new TimeWindow(start, end); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs new file mode 100644 index 00000000..9ae6106b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs @@ -0,0 +1,84 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Common.Cursors; + +/// +/// Represents the persisted state of a sliding time-window cursor. +/// +public sealed record TimeWindowCursorState(DateTimeOffset? LastWindowStart, DateTimeOffset? LastWindowEnd) +{ + public static TimeWindowCursorState Empty { get; } = new(null, null); + + public TimeWindowCursorState WithWindow(TimeWindow window) + { + return new TimeWindowCursorState(window.Start, window.End); + } + + public BsonDocument ToBsonDocument(string startField = "windowStart", string endField = "windowEnd") + { + var document = new BsonDocument(); + WriteTo(document, startField, endField); + return document; + } + + public void WriteTo(BsonDocument document, string startField = "windowStart", string endField = "windowEnd") + { + ArgumentNullException.ThrowIfNull(document); + ArgumentException.ThrowIfNullOrEmpty(startField); + ArgumentException.ThrowIfNullOrEmpty(endField); + + document.Remove(startField); + document.Remove(endField); + + if (LastWindowStart.HasValue) + { + document[startField] = LastWindowStart.Value.UtcDateTime; + } + + if (LastWindowEnd.HasValue) + { + document[endField] = LastWindowEnd.Value.UtcDateTime; + } + } + + public static TimeWindowCursorState FromBsonDocument(BsonDocument? document, string startField = "windowStart", string endField = "windowEnd") + { + if (document is null) + { + return Empty; + } + + DateTimeOffset? start = null; + DateTimeOffset? end = null; + + if (document.TryGetValue(startField, out var startValue)) + { + start = ReadDateTimeOffset(startValue); + } + + if (document.TryGetValue(endField, out var endValue)) + { + end = ReadDateTimeOffset(endValue); + } + + return new TimeWindowCursorState(start, end); + } + + private static DateTimeOffset? ReadDateTimeOffset(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } +} + +/// +/// Simple value object describing a time window. +/// +public readonly record struct TimeWindow(DateTimeOffset Start, DateTimeOffset End) +{ + public TimeSpan Duration => End - Start; +} diff --git a/src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs b/src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs new file mode 100644 index 00000000..25055139 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs @@ -0,0 +1,27 @@ +namespace StellaOps.Feedser.Source.Common; + +/// +/// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages. +/// +public static class DocumentStatuses +{ + /// + /// Document captured from the upstream source and awaiting schema validation/parsing. + /// + public const string PendingParse = "pending-parse"; + + /// + /// Document parsed and sanitized; awaiting canonical mapping. + /// + public const string PendingMap = "pending-map"; + + /// + /// Document fully mapped to canonical advisories. + /// + public const string Mapped = "mapped"; + + /// + /// Document failed processing; requires manual intervention before retry. + /// + public const string Failed = "failed"; +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs b/src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs new file mode 100644 index 00000000..7c67ccc2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs @@ -0,0 +1,43 @@ +using System.Security.Cryptography; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Jitter source backed by for thread-safe, high-entropy delays. +/// +public sealed class CryptoJitterSource : IJitterSource +{ + public TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive) + { + if (maxInclusive < minInclusive) + { + throw new ArgumentException("Max jitter must be greater than or equal to min jitter.", nameof(maxInclusive)); + } + + if (minInclusive < TimeSpan.Zero) + { + minInclusive = TimeSpan.Zero; + } + + if (maxInclusive == minInclusive) + { + return minInclusive; + } + + var minTicks = minInclusive.Ticks; + var maxTicks = maxInclusive.Ticks; + var range = maxTicks - minTicks; + + Span buffer = stackalloc byte[8]; + RandomNumberGenerator.Fill(buffer); + var sample = BitConverter.ToUInt64(buffer); + var ratio = sample / (double)ulong.MaxValue; + var jitterTicks = (long)Math.Round(range * ratio, MidpointRounding.AwayFromZero); + if (jitterTicks > range) + { + jitterTicks = range; + } + + return TimeSpan.FromTicks(minTicks + jitterTicks); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs b/src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs new file mode 100644 index 00000000..b6b7a2f7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Produces random jitter durations used to decorrelate retries. +/// +public interface IJitterSource +{ + TimeSpan Next(TimeSpan minInclusive, TimeSpan maxInclusive); +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs b/src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs new file mode 100644 index 00000000..bc3fc7b1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs @@ -0,0 +1,90 @@ +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Handles persistence of raw upstream documents in GridFS buckets for later parsing. +/// +public sealed class RawDocumentStorage +{ + private const string BucketName = "documents"; + + private readonly IMongoDatabase _database; + + public RawDocumentStorage(IMongoDatabase database) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + } + + private GridFSBucket CreateBucket() => new(_database, new GridFSBucketOptions + { + BucketName = BucketName, + WriteConcern = _database.Settings.WriteConcern, + ReadConcern = _database.Settings.ReadConcern, + }); + + public Task UploadAsync( + string sourceName, + string uri, + byte[] content, + string? contentType, + CancellationToken cancellationToken) + => UploadAsync(sourceName, uri, content, contentType, expiresAt: null, cancellationToken); + + public async Task UploadAsync( + string sourceName, + string uri, + byte[] content, + string? contentType, + DateTimeOffset? expiresAt, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(uri); + ArgumentNullException.ThrowIfNull(content); + + var bucket = CreateBucket(); + var filename = $"{sourceName}/{Guid.NewGuid():N}"; + var metadata = new BsonDocument + { + ["sourceName"] = sourceName, + ["uri"] = uri, + }; + + if (!string.IsNullOrWhiteSpace(contentType)) + { + metadata["contentType"] = contentType; + } + + if (expiresAt.HasValue) + { + metadata["expiresAt"] = expiresAt.Value.UtcDateTime; + } + + return await bucket.UploadFromBytesAsync(filename, content, new GridFSUploadOptions + { + Metadata = metadata, + }, cancellationToken).ConfigureAwait(false); + } + + public Task DownloadAsync(ObjectId id, CancellationToken cancellationToken) + { + var bucket = CreateBucket(); + return bucket.DownloadAsBytesAsync(id, cancellationToken: cancellationToken); + } + + public async Task DeleteAsync(ObjectId id, CancellationToken cancellationToken) + { + var bucket = CreateBucket(); + try + { + await bucket.DeleteAsync(id, cancellationToken).ConfigureAwait(false); + } + catch (GridFSFileNotFoundException) + { + // Already removed; ignore. + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs new file mode 100644 index 00000000..92b6f2ad --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs @@ -0,0 +1,58 @@ +using System.Net; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Result of fetching raw response content without persisting a document. +/// +public sealed record SourceFetchContentResult +{ + private SourceFetchContentResult( + HttpStatusCode statusCode, + byte[]? content, + bool notModified, + string? etag, + DateTimeOffset? lastModified, + string? contentType, + int attempts) + { + StatusCode = statusCode; + Content = content; + IsNotModified = notModified; + ETag = etag; + LastModified = lastModified; + ContentType = contentType; + Attempts = attempts; + } + + public HttpStatusCode StatusCode { get; } + + public byte[]? Content { get; } + + public bool IsSuccess => Content is not null; + + public bool IsNotModified { get; } + + public string? ETag { get; } + + public DateTimeOffset? LastModified { get; } + + public string? ContentType { get; } + + public int Attempts { get; } + + public static SourceFetchContentResult Success( + HttpStatusCode statusCode, + byte[] content, + string? etag, + DateTimeOffset? lastModified, + string? contentType, + int attempts) + => new(statusCode, content, notModified: false, etag, lastModified, contentType, attempts); + + public static SourceFetchContentResult NotModified(HttpStatusCode statusCode, int attempts) + => new(statusCode, null, notModified: true, etag: null, lastModified: null, contentType: null, attempts); + + public static SourceFetchContentResult Skipped(HttpStatusCode statusCode, int attempts) + => new(statusCode, null, notModified: false, etag: null, lastModified: null, contentType: null, attempts); +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs new file mode 100644 index 00000000..a204f8cd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; +using System.Net.Http; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Parameters describing a fetch operation for a source connector. +/// +public sealed record SourceFetchRequest( + string ClientName, + string SourceName, + HttpMethod Method, + Uri RequestUri, + IReadOnlyDictionary? Metadata = null, + string? ETag = null, + DateTimeOffset? LastModified = null, + TimeSpan? TimeoutOverride = null, + IReadOnlyList? AcceptHeaders = null) +{ + public SourceFetchRequest(string clientName, string sourceName, Uri requestUri) + : this(clientName, sourceName, HttpMethod.Get, requestUri) + { + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs new file mode 100644 index 00000000..b1c7afd8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs @@ -0,0 +1,34 @@ +using System.Net; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Outcome of fetching a raw document from an upstream source. +/// +public sealed record SourceFetchResult +{ + private SourceFetchResult(HttpStatusCode statusCode, DocumentRecord? document, bool notModified) + { + StatusCode = statusCode; + Document = document; + IsNotModified = notModified; + } + + public HttpStatusCode StatusCode { get; } + + public DocumentRecord? Document { get; } + + public bool IsSuccess => Document is not null; + + public bool IsNotModified { get; } + + public static SourceFetchResult Success(DocumentRecord document, HttpStatusCode statusCode) + => new(statusCode, document, notModified: false); + + public static SourceFetchResult NotModified(HttpStatusCode statusCode) + => new(statusCode, null, notModified: true); + + public static SourceFetchResult Skipped(HttpStatusCode statusCode) + => new(statusCode, null, notModified: false); +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs new file mode 100644 index 00000000..32972895 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs @@ -0,0 +1,313 @@ +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Telemetry; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Executes HTTP fetches for connectors, capturing raw responses with metadata for downstream stages. +/// +public sealed class SourceFetchService +{ + private static readonly string[] DefaultAcceptHeaders = { "application/json" }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly IOptionsMonitor _httpClientOptions; + private readonly IOptions _storageOptions; + private readonly IJitterSource _jitterSource; + + public SourceFetchService( + IHttpClientFactory httpClientFactory, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + ILogger logger, + IJitterSource jitterSource, + TimeProvider? timeProvider = null, + IOptionsMonitor? httpClientOptions = null, + IOptions? storageOptions = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _jitterSource = jitterSource ?? throw new ArgumentNullException(nameof(jitterSource)); + _timeProvider = timeProvider ?? TimeProvider.System; + _httpClientOptions = httpClientOptions ?? throw new ArgumentNullException(nameof(httpClientOptions)); + _storageOptions = storageOptions ?? throw new ArgumentNullException(nameof(storageOptions)); + } + + public async Task FetchAsync(SourceFetchRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + using var activity = SourceDiagnostics.StartFetch(request.SourceName, request.RequestUri, request.Method.Method, request.ClientName); + var stopwatch = Stopwatch.StartNew(); + + try + { + var sendResult = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var response = sendResult.Response; + + using (response) + { + var duration = stopwatch.Elapsed; + activity?.SetTag("http.status_code", (int)response.StatusCode); + activity?.SetTag("http.retry.count", sendResult.Attempts - 1); + + var rateLimitRemaining = TryGetHeaderValue(response.Headers, "x-ratelimit-remaining"); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + _logger.LogDebug("Source {Source} returned 304 Not Modified for {Uri}", request.SourceName, request.RequestUri); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, rateLimitRemaining); + activity?.SetStatus(ActivityStatusCode.Ok); + return SourceFetchResult.NotModified(response.StatusCode); + } + + if (!response.IsSuccessStatusCode) + { + var body = await ReadResponsePreviewAsync(response, cancellationToken).ConfigureAwait(false); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, rateLimitRemaining); + activity?.SetStatus(ActivityStatusCode.Error, body); + throw new HttpRequestException($"Fetch failed with status {(int)response.StatusCode} {response.StatusCode} from {request.RequestUri}. Body preview: {body}"); + } + + var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var sha256 = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant(); + var fetchedAt = _timeProvider.GetUtcNow(); + var contentType = response.Content.Headers.ContentType?.ToString(); + var storageOptions = _storageOptions.Value; + var retention = storageOptions.RawDocumentRetention; + DateTimeOffset? expiresAt = null; + if (retention > TimeSpan.Zero) + { + var grace = storageOptions.RawDocumentRetentionTtlGrace >= TimeSpan.Zero + ? storageOptions.RawDocumentRetentionTtlGrace + : TimeSpan.Zero; + + try + { + expiresAt = fetchedAt.Add(retention).Add(grace); + } + catch (ArgumentOutOfRangeException) + { + expiresAt = DateTimeOffset.MaxValue; + } + } + + var gridFsId = await _rawDocumentStorage.UploadAsync( + request.SourceName, + request.RequestUri.ToString(), + contentBytes, + contentType, + expiresAt, + cancellationToken).ConfigureAwait(false); + + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var header in response.Headers) + { + headers[header.Key] = string.Join(",", header.Value); + } + + foreach (var header in response.Content.Headers) + { + headers[header.Key] = string.Join(",", header.Value); + } + + var metadata = request.Metadata is null + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(request.Metadata, StringComparer.Ordinal); + metadata["attempts"] = sendResult.Attempts.ToString(CultureInfo.InvariantCulture); + metadata["fetchedAt"] = fetchedAt.ToString("O"); + + var existing = await _documentStore.FindBySourceAndUriAsync(request.SourceName, request.RequestUri.ToString(), cancellationToken).ConfigureAwait(false); + var recordId = existing?.Id ?? Guid.NewGuid(); + + var record = new DocumentRecord( + recordId, + request.SourceName, + request.RequestUri.ToString(), + fetchedAt, + sha256, + DocumentStatuses.PendingParse, + contentType, + headers, + metadata, + response.Headers.ETag?.Tag, + response.Content.Headers.LastModified, + gridFsId, + expiresAt); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, contentBytes.LongLength, rateLimitRemaining); + activity?.SetStatus(ActivityStatusCode.Ok); + _logger.LogInformation("Fetched {Source} document {Uri} (sha256={Sha})", request.SourceName, request.RequestUri, sha256); + return SourceFetchResult.Success(upserted, response.StatusCode); + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + throw; + } + } + + public async Task FetchContentAsync(SourceFetchRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + using var activity = SourceDiagnostics.StartFetch(request.SourceName, request.RequestUri, request.Method.Method, request.ClientName); + var stopwatch = Stopwatch.StartNew(); + + try + { + _ = _httpClientOptions.Get(request.ClientName); + var sendResult = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var response = sendResult.Response; + + using (response) + { + var duration = stopwatch.Elapsed; + activity?.SetTag("http.status_code", (int)response.StatusCode); + activity?.SetTag("http.retry.count", sendResult.Attempts - 1); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + _logger.LogDebug("Source {Source} returned 304 Not Modified for {Uri}", request.SourceName, request.RequestUri); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, null); + activity?.SetStatus(ActivityStatusCode.Ok); + return SourceFetchContentResult.NotModified(response.StatusCode, sendResult.Attempts); + } + + if (!response.IsSuccessStatusCode) + { + var body = await ReadResponsePreviewAsync(response, cancellationToken).ConfigureAwait(false); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength, null); + activity?.SetStatus(ActivityStatusCode.Error, body); + throw new HttpRequestException($"Fetch failed with status {(int)response.StatusCode} {response.StatusCode} from {request.RequestUri}. Body preview: {body}"); + } + + var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + SourceDiagnostics.RecordHttpRequest(request.SourceName, request.ClientName, response.StatusCode, sendResult.Attempts, duration, response.Content.Headers.ContentLength ?? contentBytes.LongLength, null); + activity?.SetStatus(ActivityStatusCode.Ok); + return SourceFetchContentResult.Success( + response.StatusCode, + contentBytes, + response.Headers.ETag?.Tag, + response.Content.Headers.LastModified, + response.Content.Headers.ContentType?.ToString(), + sendResult.Attempts); + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + throw; + } + } + + private async Task SendAsync(SourceFetchRequest request, HttpCompletionOption completionOption, CancellationToken cancellationToken) + { + var attemptCount = 0; + var options = _httpClientOptions.Get(request.ClientName); + + var response = await SourceRetryPolicy.SendWithRetryAsync( + () => CreateHttpRequestMessage(request), + async (httpRequest, ct) => + { + attemptCount++; + var client = _httpClientFactory.CreateClient(request.ClientName); + if (request.TimeoutOverride.HasValue) + { + client.Timeout = request.TimeoutOverride.Value; + } + + return await client.SendAsync(httpRequest, completionOption, ct).ConfigureAwait(false); + }, + maxAttempts: options.MaxAttempts, + baseDelay: options.BaseDelay, + _jitterSource, + context => SourceDiagnostics.RecordRetry( + request.SourceName, + request.ClientName, + context.Response?.StatusCode, + context.Attempt, + context.Delay), + cancellationToken).ConfigureAwait(false); + + return new SourceFetchSendResult(response, attemptCount); + } + + internal static HttpRequestMessage CreateHttpRequestMessage(SourceFetchRequest request) + { + var httpRequest = new HttpRequestMessage(request.Method, request.RequestUri); + var acceptValues = request.AcceptHeaders is { Count: > 0 } headers + ? headers + : DefaultAcceptHeaders; + + httpRequest.Headers.Accept.Clear(); + var added = false; + foreach (var mediaType in acceptValues) + { + if (string.IsNullOrWhiteSpace(mediaType)) + { + continue; + } + + if (MediaTypeWithQualityHeaderValue.TryParse(mediaType, out var headerValue)) + { + httpRequest.Headers.Accept.Add(headerValue); + added = true; + } + } + + if (!added) + { + httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(DefaultAcceptHeaders[0])); + } + + return httpRequest; + } + + private static async Task ReadResponsePreviewAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + try + { + var buffer = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var preview = Encoding.UTF8.GetString(buffer); + return preview.Length > 256 ? preview[..256] : preview; + } + catch + { + return ""; + } + } + + private static string? TryGetHeaderValue(HttpResponseHeaders headers, string name) + { + if (headers.TryGetValues(name, out var values)) + { + return values.FirstOrDefault(); + } + + return null; + } + + private readonly record struct SourceFetchSendResult(HttpResponseMessage Response, int Attempts); +} diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs b/src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs new file mode 100644 index 00000000..79c24ce3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs @@ -0,0 +1,79 @@ +namespace StellaOps.Feedser.Source.Common.Fetch; + +/// +/// Provides retry/backoff behavior for source HTTP fetches. +/// +internal static class SourceRetryPolicy +{ + public static async Task SendWithRetryAsync( + Func requestFactory, + Func> sender, + int maxAttempts, + TimeSpan baseDelay, + IJitterSource jitterSource, + Action? onRetry, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(requestFactory); + ArgumentNullException.ThrowIfNull(sender); + ArgumentNullException.ThrowIfNull(jitterSource); + + var attempt = 0; + + while (true) + { + attempt++; + using var request = requestFactory(); + HttpResponseMessage response; + + try + { + response = await sender(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < maxAttempts) + { + var delay = ComputeDelay(baseDelay, attempt, jitterSource: jitterSource); + onRetry?.Invoke(new SourceRetryAttemptContext(attempt, null, ex, delay)); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + continue; + } + + if (NeedsRetry(response) && attempt < maxAttempts) + { + var delay = ComputeDelay(baseDelay, attempt, response.Headers.RetryAfter?.Delta, jitterSource); + onRetry?.Invoke(new SourceRetryAttemptContext(attempt, response, null, delay)); + response.Dispose(); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + continue; + } + + return response; + } + } + + private static bool NeedsRetry(HttpResponseMessage response) + { + if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + { + return true; + } + + var status = (int)response.StatusCode; + return status >= 500 && status < 600; + } + + private static TimeSpan ComputeDelay(TimeSpan baseDelay, int attempt, TimeSpan? retryAfter = null, IJitterSource? jitterSource = null) + { + if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero) + { + return retryAfter.Value; + } + + var exponential = TimeSpan.FromMilliseconds(baseDelay.TotalMilliseconds * Math.Pow(2, attempt - 1)); + var jitter = jitterSource?.Next(TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(250)) + ?? TimeSpan.FromMilliseconds(Random.Shared.Next(50, 250)); + return exponential + jitter; + } +} + +internal readonly record struct SourceRetryAttemptContext(int Attempt, HttpResponseMessage? Response, Exception? Exception, TimeSpan Delay); diff --git a/src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs b/src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs new file mode 100644 index 00000000..2875cc32 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs @@ -0,0 +1,168 @@ +using System.Linq; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; +using StellaOps.Feedser.Source.Common.Url; + +namespace StellaOps.Feedser.Source.Common.Html; + +/// +/// Sanitizes untrusted HTML fragments produced by upstream advisories. +/// Removes executable content, enforces an allowlist of elements, and normalizes anchor href values. +/// +public sealed class HtmlContentSanitizer +{ + private static readonly HashSet AllowedElements = new(StringComparer.OrdinalIgnoreCase) + { + "a", "abbr", "b", "blockquote", "br", "code", "dd", "dl", "dt", + "em", "i", "li", "ol", "p", "pre", "s", "small", "span", + "strong", "sub", "sup", "table", "tbody", "td", "th", "thead", "tr", "ul" + }; + + private static readonly HashSet UrlAttributes = new(StringComparer.OrdinalIgnoreCase) + { + "href", "src", + }; + + private readonly HtmlParser _parser; + + public HtmlContentSanitizer() + { + _parser = new HtmlParser(new HtmlParserOptions + { + IsKeepingSourceReferences = false, + }); + } + + /// + /// Sanitizes and returns a safe fragment suitable for rendering. + /// + public string Sanitize(string? html, Uri? baseUri = null) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + var document = _parser.ParseDocument(html); + if (document.Body is null) + { + return string.Empty; + } + + foreach (var element in document.All.ToList()) + { + if (IsDangerous(element)) + { + element.Remove(); + continue; + } + + if (!AllowedElements.Contains(element.LocalName)) + { + var owner = element.Owner; + if (owner is null) + { + element.Remove(); + continue; + } + + var text = element.TextContent ?? string.Empty; + element.Replace(owner.CreateTextNode(text)); + continue; + } + + CleanAttributes(element, baseUri); + } + + return document.Body.InnerHtml.Trim(); + } + + private static bool IsDangerous(IElement element) + { + if (string.Equals(element.LocalName, "script", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "style", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "iframe", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "object", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "embed", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + + private static void CleanAttributes(IElement element, Uri? baseUri) + { + foreach (var attribute in element.Attributes.ToList()) + { + if (attribute.Name.StartsWith("on", StringComparison.OrdinalIgnoreCase)) + { + element.RemoveAttribute(attribute.Name); + continue; + } + + if (UrlAttributes.Contains(attribute.Name)) + { + NormalizeUrlAttribute(element, attribute, baseUri); + continue; + } + + if (!IsAttributeAllowed(element.LocalName, attribute.Name)) + { + element.RemoveAttribute(attribute.Name); + } + } + } + + private static bool IsAttributeAllowed(string elementName, string attributeName) + { + if (string.Equals(attributeName, "title", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(elementName, "a", StringComparison.OrdinalIgnoreCase) + && string.Equals(attributeName, "rel", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(elementName, "table", StringComparison.OrdinalIgnoreCase) + && (string.Equals(attributeName, "border", StringComparison.OrdinalIgnoreCase) + || string.Equals(attributeName, "cellpadding", StringComparison.OrdinalIgnoreCase) + || string.Equals(attributeName, "cellspacing", StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + return false; + } + + private static void NormalizeUrlAttribute(IElement element, IAttr attribute, Uri? baseUri) + { + if (string.IsNullOrWhiteSpace(attribute.Value)) + { + element.RemoveAttribute(attribute.Name); + return; + } + + if (!UrlNormalizer.TryNormalize(attribute.Value, baseUri, out var normalized)) + { + element.RemoveAttribute(attribute.Name); + return; + } + + if (string.Equals(element.LocalName, "a", StringComparison.OrdinalIgnoreCase)) + { + element.SetAttribute("rel", "noopener nofollow noreferrer"); + } + + if (normalized is null) + { + element.RemoveAttribute(attribute.Name); + return; + } + + element.SetAttribute(attribute.Name, normalized.ToString()); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs b/src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs new file mode 100644 index 00000000..905e7ce7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs @@ -0,0 +1,36 @@ +using System.Net.Http.Headers; + +namespace StellaOps.Feedser.Source.Common.Http; + +/// +/// Delegating handler that enforces an allowlist of destination hosts for outbound requests. +/// +internal sealed class AllowlistedHttpMessageHandler : DelegatingHandler +{ + private readonly IReadOnlyCollection _allowedHosts; + + public AllowlistedHttpMessageHandler(SourceHttpClientOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var snapshot = options.GetAllowedHostsSnapshot(); + if (snapshot.Count == 0) + { + throw new InvalidOperationException("Source HTTP client must configure at least one allowed host."); + } + + _allowedHosts = snapshot; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var host = request.RequestUri?.Host; + if (string.IsNullOrWhiteSpace(host) || !_allowedHosts.Contains(host)) + { + throw new InvalidOperationException($"Request host '{host ?? ""}' is not allowlisted for this source."); + } + + return base.SendAsync(request, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..39a536ac --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Xml; + +namespace StellaOps.Feedser.Source.Common.Http; + +public static class ServiceCollectionExtensions +{ + /// + /// Registers a named HTTP client configured for a source connector with allowlisted hosts and sensible defaults. + /// + public static IHttpClientBuilder AddSourceHttpClient(this IServiceCollection services, string name, Action configure) + => services.AddSourceHttpClient(name, (_, options) => configure(options)); + + public static IHttpClientBuilder AddSourceHttpClient(this IServiceCollection services, string name, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions(name).Configure((options, sp) => configure(sp, options)); + + return services + .AddHttpClient(name) + .ConfigureHttpClient((sp, client) => + { + var options = sp.GetRequiredService>().Get(name); + + if (options.BaseAddress is not null) + { + client.BaseAddress = options.BaseAddress; + } + + client.Timeout = options.Timeout; + client.DefaultRequestHeaders.UserAgent.Clear(); + client.DefaultRequestHeaders.UserAgent.ParseAdd(options.UserAgent); + + foreach (var header in options.DefaultRequestHeaders) + { + client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value); + } + }) + .ConfigurePrimaryHttpMessageHandler((sp) => + { + var options = sp.GetRequiredService>().Get(name).Clone(); + return new HttpClientHandler + { + AllowAutoRedirect = options.AllowAutoRedirect, + AutomaticDecompression = System.Net.DecompressionMethods.All, + }; + }) + .AddHttpMessageHandler(sp => + { + var options = sp.GetRequiredService>().Get(name).Clone(); + return new AllowlistedHttpMessageHandler(options); + }); + } + + /// + /// Registers shared helpers used by source connectors. + /// + public static IServiceCollection AddSourceCommon(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs b/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs new file mode 100644 index 00000000..b9f9b165 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs @@ -0,0 +1,80 @@ +using System.Collections.ObjectModel; + +namespace StellaOps.Feedser.Source.Common.Http; + +/// +/// Configuration applied to named HTTP clients used by connectors. +/// +public sealed class SourceHttpClientOptions +{ + private readonly HashSet _allowedHosts = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _defaultHeaders = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Gets or sets the base address used for relative requests. + /// + public Uri? BaseAddress { get; set; } + + /// + /// Gets or sets the client timeout. + /// + public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets the user-agent string applied to outgoing requests. + /// + public string UserAgent { get; set; } = "StellaOps.Feedser/1.0"; + + /// + /// Gets or sets whether redirects are allowed. Defaults to true. + /// + public bool AllowAutoRedirect { get; set; } = true; + + /// + /// Maximum number of retry attempts for transient failures. + /// + public int MaxAttempts { get; set; } = 3; + + /// + /// Base delay applied to the exponential backoff policy. + /// + public TimeSpan BaseDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Hosts that this client is allowed to contact. + /// + public ISet AllowedHosts => _allowedHosts; + + /// + /// Default request headers appended to each outgoing request. + /// + public IDictionary DefaultRequestHeaders => _defaultHeaders; + + internal SourceHttpClientOptions Clone() + { + var clone = new SourceHttpClientOptions + { + BaseAddress = BaseAddress, + Timeout = Timeout, + UserAgent = UserAgent, + AllowAutoRedirect = AllowAutoRedirect, + MaxAttempts = MaxAttempts, + BaseDelay = BaseDelay, + }; + + foreach (var host in _allowedHosts) + { + clone.AllowedHosts.Add(host); + } + + foreach (var header in _defaultHeaders) + { + clone.DefaultRequestHeaders[header.Key] = header.Value; + } + + return clone; + } + + internal IReadOnlyCollection GetAllowedHostsSnapshot() + => new ReadOnlyCollection(_allowedHosts.ToArray()); +} diff --git a/src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs b/src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs new file mode 100644 index 00000000..50850b9b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs @@ -0,0 +1,9 @@ +using System.Text.Json; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Common.Json; + +public interface IJsonSchemaValidator +{ + void Validate(JsonDocument document, JsonSchema schema, string documentName); +} diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs new file mode 100644 index 00000000..07010318 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Feedser.Source.Common.Json; + +public sealed record JsonSchemaValidationError( + string InstanceLocation, + string SchemaLocation, + string Message, + string Keyword); diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs new file mode 100644 index 00000000..9842f5e2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Feedser.Source.Common.Json; + +public sealed class JsonSchemaValidationException : Exception +{ + public JsonSchemaValidationException(string documentName, IReadOnlyList errors) + : base($"JSON schema validation failed for '{documentName}'.") + { + DocumentName = documentName; + Errors = errors ?? Array.Empty(); + } + + public string DocumentName { get; } + + public IReadOnlyList Errors { get; } +} diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs new file mode 100644 index 00000000..5b537325 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Json.Schema; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Source.Common.Json; +public sealed class JsonSchemaValidator : IJsonSchemaValidator +{ + private readonly ILogger _logger; + private const int MaxLoggedErrors = 5; + + public JsonSchemaValidator(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void Validate(JsonDocument document, JsonSchema schema, string documentName) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(schema); + ArgumentException.ThrowIfNullOrEmpty(documentName); + + var result = schema.Evaluate(document.RootElement, new EvaluationOptions + { + OutputFormat = OutputFormat.List, + RequireFormatValidation = true, + }); + + if (result.IsValid) + { + return; + } + + var errors = CollectErrors(result); + + if (errors.Count == 0) + { + _logger.LogWarning("Schema validation failed for {Document} with unknown errors", documentName); + throw new JsonSchemaValidationException(documentName, errors); + } + + foreach (var violation in errors.Take(MaxLoggedErrors)) + { + _logger.LogWarning( + "Schema violation for {Document} at {InstanceLocation} (keyword: {Keyword}): {Message}", + documentName, + string.IsNullOrEmpty(violation.InstanceLocation) ? "#" : violation.InstanceLocation, + violation.Keyword, + violation.Message); + } + + if (errors.Count > MaxLoggedErrors) + { + _logger.LogWarning("{Count} additional schema violations for {Document} suppressed", errors.Count - MaxLoggedErrors, documentName); + } + + throw new JsonSchemaValidationException(documentName, errors); + } + + private static IReadOnlyList CollectErrors(EvaluationResults result) + { + var errors = new List(); + Aggregate(result, errors); + return errors; + } + + private static void Aggregate(EvaluationResults node, List errors) + { + if (node.Errors is { Count: > 0 }) + { + foreach (var kvp in node.Errors) + { + errors.Add(new JsonSchemaValidationError( + node.InstanceLocation?.ToString() ?? string.Empty, + node.SchemaLocation?.ToString() ?? string.Empty, + kvp.Value, + kvp.Key)); + } + } + + if (node.Details is null) + { + return; + } + + foreach (var child in node.Details) + { + Aggregate(child, errors); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs b/src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs new file mode 100644 index 00000000..cf09e600 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs @@ -0,0 +1,142 @@ +using System.Linq; +using System.Text; +using NuGet.Versioning; +using StellaOps.Feedser.Normalization.Identifiers; + +namespace StellaOps.Feedser.Source.Common.Packages; + +/// +/// Shared helpers for working with Package URLs and SemVer coordinates inside connectors. +/// +public static class PackageCoordinateHelper +{ + public static bool TryParsePackageUrl(string? value, out PackageCoordinates? coordinates) + { + coordinates = null; + if (!IdentifierNormalizer.TryNormalizePackageUrl(value, out var canonical, out var packageUrl) || packageUrl is null) + { + return false; + } + + var qualifiers = packageUrl.Qualifiers.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.OrdinalIgnoreCase); + coordinates = new PackageCoordinates( + Canonical: canonical!, + Type: packageUrl.Type, + NamespaceSegments: packageUrl.NamespaceSegments.ToArray(), + Name: packageUrl.Name, + Version: packageUrl.Version, + Qualifiers: qualifiers, + SubpathSegments: packageUrl.SubpathSegments.ToArray(), + Original: packageUrl.Original); + return true; + } + + public static PackageCoordinates ParsePackageUrl(string value) + { + if (!TryParsePackageUrl(value, out var coordinates) || coordinates is null) + { + throw new FormatException($"Value '{value}' is not a valid Package URL"); + } + + return coordinates; + } + + public static bool TryParseSemVer(string? value, out SemanticVersion? version, out string? normalized) + { + version = null; + normalized = null; + + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (!SemanticVersion.TryParse(value.Trim(), out var parsed)) + { + return false; + } + + version = parsed; + normalized = parsed.ToNormalizedString(); + return true; + } + + public static bool TryParseSemVerRange(string? value, out VersionRange? range) + { + range = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (!VersionRange.TryParse(value.Trim(), out var parsed)) + { + return false; + } + + range = parsed; + return true; + } + + public static string BuildPackageUrl( + string type, + IReadOnlyList? namespaceSegments, + string name, + string? version = null, + IReadOnlyDictionary? qualifiers = null, + IReadOnlyList? subpathSegments = null) + { + ArgumentException.ThrowIfNullOrEmpty(type); + ArgumentException.ThrowIfNullOrEmpty(name); + + var builder = new StringBuilder("pkg:"); + builder.Append(type.Trim().ToLowerInvariant()); + builder.Append('/'); + + if (namespaceSegments is not null && namespaceSegments.Count > 0) + { + builder.Append(string.Join('/', namespaceSegments.Select(NormalizeSegment))); + builder.Append('/'); + } + + builder.Append(NormalizeSegment(name)); + + if (!string.IsNullOrWhiteSpace(version)) + { + builder.Append('@'); + builder.Append(version.Trim()); + } + + if (qualifiers is not null && qualifiers.Count > 0) + { + builder.Append('?'); + builder.Append(string.Join('&', qualifiers + .OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase) + .Select(kvp => $"{NormalizeSegment(kvp.Key)}={NormalizeSegment(kvp.Value)}"))); + } + + if (subpathSegments is not null && subpathSegments.Count > 0) + { + builder.Append('#'); + builder.Append(string.Join('/', subpathSegments.Select(NormalizeSegment))); + } + + return builder.ToString(); + } + + private static string NormalizeSegment(string value) + { + ArgumentNullException.ThrowIfNull(value); + return Uri.EscapeDataString(value.Trim()); + } +} + +public sealed record PackageCoordinates( + string Canonical, + string Type, + IReadOnlyList NamespaceSegments, + string Name, + string? Version, + IReadOnlyDictionary Qualifiers, + IReadOnlyList SubpathSegments, + string Original); diff --git a/src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs b/src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs new file mode 100644 index 00000000..8fc2b07c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs @@ -0,0 +1,103 @@ +using System.Collections.Generic; +using System.Text; +using UglyToad.PdfPig; +using UglyToad.PdfPig.Content; + +namespace StellaOps.Feedser.Source.Common.Pdf; + +/// +/// Extracts text from PDF advisories using UglyToad.PdfPig without requiring native dependencies. +/// +public sealed class PdfTextExtractor +{ + public async Task ExtractTextAsync(Stream pdfStream, PdfExtractionOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(pdfStream); + options ??= PdfExtractionOptions.Default; + + using var buffer = new MemoryStream(); + await pdfStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + buffer.Position = 0; + + using var document = PdfDocument.Open(buffer, new ParsingOptions + { + ClipPaths = true, + UseLenientParsing = true, + }); + + var builder = new StringBuilder(); + var pageCount = 0; + + foreach (var page in document.GetPages()) + { + cancellationToken.ThrowIfCancellationRequested(); + + pageCount++; + if (options.MaxPages.HasValue && pageCount > options.MaxPages.Value) + { + break; + } + + if (pageCount > 1 && options.PageSeparator is not null) + { + builder.Append(options.PageSeparator); + } + + var text = options.PreserveLayout + ? page.Text + : FlattenWords(page.GetWords()); + + if (!string.IsNullOrWhiteSpace(text)) + { + builder.AppendLine(text.Trim()); + } + } + + return new PdfExtractionResult(builder.ToString().Trim(), pageCount); + } + + private static string FlattenWords(IEnumerable words) + { + var builder = new StringBuilder(); + var first = true; + foreach (var word in words) + { + if (string.IsNullOrWhiteSpace(word.Text)) + { + continue; + } + + if (!first) + { + builder.Append(' '); + } + + builder.Append(word.Text.Trim()); + first = false; + } + + return builder.ToString(); + } +} + +public sealed record PdfExtractionResult(string Text, int PagesProcessed); + +public sealed record PdfExtractionOptions +{ + public static PdfExtractionOptions Default { get; } = new(); + + /// + /// Maximum number of pages to read. Null reads the entire document. + /// + public int? MaxPages { get; init; } + + /// + /// When true, uses PdfPig's native layout text. When false, collapses to a single line per page. + /// + public bool PreserveLayout { get; init; } = true; + + /// + /// Separator inserted between pages. Null disables separators. + /// + public string? PageSeparator { get; init; } = "\n\n"; +} diff --git a/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..d6e4b5d8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Common.Tests")] diff --git a/src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj b/src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj new file mode 100644 index 00000000..272e2054 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj @@ -0,0 +1,21 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Common/TASKS.md b/src/StellaOps.Feedser.Source.Common/TASKS.md new file mode 100644 index 00000000..94717178 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/TASKS.md @@ -0,0 +1,16 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Register source HTTP clients with allowlists and timeouts|BE-Conn-Shared|Source.Common|**DONE** – `AddSourceHttpClient` wires named clients with host allowlists/timeouts.| +|Implement retry/backoff with jitter and 429 handling|BE-Conn-Shared|Source.Common|**DONE** – `SourceRetryPolicy` retries with 429/5xx handling and exponential backoff.| +|Conditional GET helpers (ETag/Last-Modified)|BE-Conn-Shared|Source.Common|**DONE** – `SourceFetchRequest` + fetch result propagate etag/last-modified for NotModified handling.| +|Windowed cursor and pagination utilities|BE-Conn-Shared|Source.Common|**DONE** – `TimeWindowCursorPlanner` + `PaginationPlanner` centralize sliding windows and additional page indices.| +|JSON/XML schema validators with rich errors|BE-Conn-Shared, QA|Source.Common|DONE – JsonSchemaValidator surfaces keyword/path/message details + tests.| +|Raw document capture helper|BE-Conn-Shared|Storage.Mongo|**DONE** – `SourceFetchService` stores raw payload + headers with sha256 metadata.| +|Canned HTTP test harness|QA|Source.Common|DONE – enriched `CannedHttpMessageHandler` with method-aware queues, request capture, fallbacks, and helpers + unit coverage.| +|HTML sanitization and URL normalization utilities|BE-Conn-Shared|Source.Common|DONE – `HtmlContentSanitizer` + `UrlNormalizer` provide safe fragments and canonical links for connectors.| +|PDF-to-text sandbox helper|BE-Conn-Shared|Source.Common|DONE – `PdfTextExtractor` uses PdfPig to yield deterministic text with options + tests.| +|PURL and SemVer helper library|BE-Conn-Shared|Models|DONE – `PackageCoordinateHelper` exposes normalized purl + SemVer parsing utilities backed by normalization.| +|Telemetry wiring (logs/metrics/traces)|BE-Conn-Shared|Observability|DONE – `SourceDiagnostics` emits Activity/Meter signals integrated into fetch pipeline and WebService OTEL setup.| +|Shared jitter source in retry policy|BE-Conn-Shared|Source.Common|**DONE** – `SourceRetryPolicy` now consumes injected `CryptoJitterSource` for thread-safe jitter.| +|Allow per-request Accept header overrides|BE-Conn-Shared|Source.Common|**DONE** – `SourceFetchRequest.AcceptHeaders` honored by `SourceFetchService` plus unit tests for overrides.| diff --git a/src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs b/src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs new file mode 100644 index 00000000..fc0276bf --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs @@ -0,0 +1,107 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Net; + +namespace StellaOps.Feedser.Source.Common.Telemetry; + +/// +/// Central telemetry instrumentation for connector HTTP operations. +/// +public static class SourceDiagnostics +{ + public const string ActivitySourceName = "StellaOps.Feedser.Source"; + public const string MeterName = "StellaOps.Feedser.Source"; + + private static readonly ActivitySource ActivitySource = new(ActivitySourceName); + private static readonly Meter Meter = new(MeterName); + + private static readonly Counter HttpRequestCounter = Meter.CreateCounter("feedser.source.http.requests"); + private static readonly Counter HttpRetryCounter = Meter.CreateCounter("feedser.source.http.retries"); + private static readonly Counter HttpFailureCounter = Meter.CreateCounter("feedser.source.http.failures"); + private static readonly Counter HttpNotModifiedCounter = Meter.CreateCounter("feedser.source.http.not_modified"); + private static readonly Histogram HttpDuration = Meter.CreateHistogram("feedser.source.http.duration", unit: "ms"); + private static readonly Histogram HttpPayloadBytes = Meter.CreateHistogram("feedser.source.http.payload_bytes", unit: "byte"); + + public static Activity? StartFetch(string sourceName, Uri requestUri, string httpMethod, string? clientName) + { + var tags = new ActivityTagsCollection + { + { "feedser.source", sourceName }, + { "http.method", httpMethod }, + { "http.url", requestUri.ToString() }, + }; + + if (!string.IsNullOrWhiteSpace(clientName)) + { + tags.Add("http.client_name", clientName!); + } + + return ActivitySource.StartActivity("SourceFetch", ActivityKind.Client, parentContext: default, tags: tags); + } + + public static void RecordHttpRequest(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount, TimeSpan duration, long? contentLength, string? rateLimitRemaining) + { + var tags = BuildDefaultTags(sourceName, clientName, statusCode, attemptCount); + HttpRequestCounter.Add(1, tags); + HttpDuration.Record(duration.TotalMilliseconds, tags); + + if (contentLength.HasValue && contentLength.Value >= 0) + { + HttpPayloadBytes.Record(contentLength.Value, tags); + } + + if (statusCode == HttpStatusCode.NotModified) + { + HttpNotModifiedCounter.Add(1, tags); + } + + if ((int)statusCode >= 500 || statusCode == HttpStatusCode.TooManyRequests) + { + HttpFailureCounter.Add(1, tags); + } + + if (!string.IsNullOrWhiteSpace(rateLimitRemaining) && long.TryParse(rateLimitRemaining, out var remaining)) + { + tags.Add("http.rate_limit.remaining", remaining); + } + } + + public static void RecordRetry(string sourceName, string? clientName, HttpStatusCode? statusCode, int attempt, TimeSpan delay) + { + var tags = new TagList + { + { "feedser.source", sourceName }, + { "http.retry_attempt", attempt }, + { "http.retry_delay_ms", delay.TotalMilliseconds }, + }; + + if (clientName is not null) + { + tags.Add("http.client_name", clientName); + } + + if (statusCode.HasValue) + { + tags.Add("http.status_code", (int)statusCode.Value); + } + + HttpRetryCounter.Add(1, tags); + } + + private static TagList BuildDefaultTags(string sourceName, string? clientName, HttpStatusCode statusCode, int attemptCount) + { + var tags = new TagList + { + { "feedser.source", sourceName }, + { "http.status_code", (int)statusCode }, + { "http.attempts", attemptCount }, + }; + + if (clientName is not null) + { + tags.Add("http.client_name", clientName); + } + + return tags; + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs b/src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs new file mode 100644 index 00000000..97db2f50 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs @@ -0,0 +1,210 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http; +using System.Text; + +namespace StellaOps.Feedser.Source.Common.Testing; + +/// +/// Deterministic HTTP handler used by tests to supply canned responses keyed by request URI and method. +/// Tracks requests for assertions and supports fallbacks/exceptions. +/// +public sealed class CannedHttpMessageHandler : HttpMessageHandler +{ + private readonly ConcurrentDictionary>> _responses = + new(RequestKeyComparer.Instance); + + private readonly ConcurrentQueue _requests = new(); + + private Func? _fallback; + + /// + /// Recorded requests in arrival order. + /// + public IReadOnlyCollection Requests => _requests.ToArray(); + + /// + /// Registers a canned response for a GET request to . + /// + public void AddResponse(Uri requestUri, Func factory) + => AddResponse(HttpMethod.Get, requestUri, _ => factory()); + + /// + /// Registers a canned response for the specified method and URI. + /// + public void AddResponse(HttpMethod method, Uri requestUri, Func factory) + => AddResponse(method, requestUri, _ => factory()); + + /// + /// Registers a canned response using the full request context. + /// + public void AddResponse(HttpMethod method, Uri requestUri, Func factory) + { + ArgumentNullException.ThrowIfNull(method); + ArgumentNullException.ThrowIfNull(requestUri); + ArgumentNullException.ThrowIfNull(factory); + + var key = new RequestKey(method, requestUri); + var queue = _responses.GetOrAdd(key, static _ => new ConcurrentQueue>()); + queue.Enqueue(factory); + } + + /// + /// Registers an exception to be thrown for the specified request. + /// + public void AddException(HttpMethod method, Uri requestUri, Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + AddResponse(method, requestUri, _ => throw exception); + } + + /// + /// Registers a fallback used when no specific response is queued for a request. + /// + public void SetFallback(Func fallback) + { + ArgumentNullException.ThrowIfNull(fallback); + _fallback = fallback; + } + + /// + /// Clears registered responses and captured requests. + /// + public void Clear() + { + _responses.Clear(); + while (_requests.TryDequeue(out _)) + { + } + _fallback = null; + } + + /// + /// Throws if any responses remain queued. + /// + public void AssertNoPendingResponses() + { + foreach (var queue in _responses.Values) + { + if (!queue.IsEmpty) + { + throw new InvalidOperationException("Not all canned responses were consumed."); + } + } + } + + /// + /// Creates an wired to this handler. + /// + public HttpClient CreateClient() + => new(this, disposeHandler: false) + { + Timeout = TimeSpan.FromSeconds(10), + }; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is null) + { + throw new InvalidOperationException("Request URI is required for canned responses."); + } + + var key = new RequestKey(request.Method ?? HttpMethod.Get, request.RequestUri); + var factory = DequeueFactory(key); + + if (factory is null) + { + if (_fallback is null) + { + throw new InvalidOperationException($"No canned response registered for {request.Method} {request.RequestUri}."); + } + + factory = _fallback; + } + + var snapshot = CaptureRequest(request); + _requests.Enqueue(snapshot); + + var response = factory(request); + response.RequestMessage ??= request; + return Task.FromResult(response); + } + + private Func? DequeueFactory(RequestKey key) + { + if (_responses.TryGetValue(key, out var queue) && queue.TryDequeue(out var factory)) + { + return factory; + } + + return null; + } + + private static CannedRequestRecord CaptureRequest(HttpRequestMessage request) + { + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var header in request.Headers) + { + headers[header.Key] = string.Join(',', header.Value); + } + + if (request.Content is not null) + { + foreach (var header in request.Content.Headers) + { + headers[header.Key] = string.Join(',', header.Value); + } + } + + return new CannedRequestRecord( + Timestamp: DateTimeOffset.UtcNow, + Method: request.Method ?? HttpMethod.Get, + Uri: request.RequestUri!, + Headers: headers); + } + + private readonly record struct RequestKey(HttpMethod Method, string Uri) + { + public RequestKey(HttpMethod method, Uri uri) + : this(method, uri.ToString()) + { + } + + public bool Equals(RequestKey other) + => string.Equals(Method.Method, other.Method.Method, StringComparison.OrdinalIgnoreCase) + && string.Equals(Uri, other.Uri, StringComparison.OrdinalIgnoreCase); + + public override int GetHashCode() + { + var methodHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Method.Method); + var uriHash = StringComparer.OrdinalIgnoreCase.GetHashCode(Uri); + return HashCode.Combine(methodHash, uriHash); + } + } + + private sealed class RequestKeyComparer : IEqualityComparer + { + public static readonly RequestKeyComparer Instance = new(); + + public bool Equals(RequestKey x, RequestKey y) => x.Equals(y); + + public int GetHashCode(RequestKey obj) => obj.GetHashCode(); + } + + public readonly record struct CannedRequestRecord(DateTimeOffset Timestamp, HttpMethod Method, Uri Uri, IReadOnlyDictionary Headers); + + private static HttpResponseMessage BuildTextResponse(HttpStatusCode statusCode, string content, string contentType) + { + var message = new HttpResponseMessage(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, contentType), + }; + return message; + } + + public void AddJsonResponse(Uri requestUri, string json, HttpStatusCode statusCode = HttpStatusCode.OK) + => AddResponse(requestUri, () => BuildTextResponse(statusCode, json, "application/json")); + + public void AddTextResponse(Uri requestUri, string content, string contentType = "text/plain", HttpStatusCode statusCode = HttpStatusCode.OK) + => AddResponse(requestUri, () => BuildTextResponse(statusCode, content, contentType)); +} diff --git a/src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs b/src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs new file mode 100644 index 00000000..1ceeebcf --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs @@ -0,0 +1,62 @@ +namespace StellaOps.Feedser.Source.Common.Url; + +/// +/// Utilities for normalizing URLs from upstream feeds. +/// +public static class UrlNormalizer +{ + /// + /// Attempts to normalize relative to . + /// Removes fragments and enforces HTTPS when possible. + /// + public static bool TryNormalize(string? value, Uri? baseUri, out Uri? normalized, bool stripFragment = true, bool forceHttps = false) + { + normalized = null; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (!Uri.TryCreate(value.Trim(), UriKind.RelativeOrAbsolute, out var candidate)) + { + return false; + } + + if (!candidate.IsAbsoluteUri) + { + if (baseUri is null) + { + return false; + } + + if (!Uri.TryCreate(baseUri, candidate, out candidate)) + { + return false; + } + } + + if (forceHttps && string.Equals(candidate.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)) + { + candidate = new UriBuilder(candidate) { Scheme = Uri.UriSchemeHttps, Port = candidate.IsDefaultPort ? -1 : candidate.Port }.Uri; + } + + if (stripFragment && !string.IsNullOrEmpty(candidate.Fragment)) + { + var builder = new UriBuilder(candidate) { Fragment = string.Empty }; + candidate = builder.Uri; + } + + normalized = candidate; + return true; + } + + public static Uri NormalizeOrThrow(string value, Uri? baseUri = null, bool stripFragment = true, bool forceHttps = false) + { + if (!TryNormalize(value, baseUri, out var normalized, stripFragment, forceHttps) || normalized is null) + { + throw new FormatException($"Value '{value}' is not a valid URI"); + } + + return normalized; + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs b/src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs new file mode 100644 index 00000000..25a6e5ea --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs @@ -0,0 +1,9 @@ +using System.Xml.Linq; +using System.Xml.Schema; + +namespace StellaOps.Feedser.Source.Common.Xml; + +public interface IXmlSchemaValidator +{ + void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName); +} diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs new file mode 100644 index 00000000..d736bc96 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Feedser.Source.Common.Xml; + +public sealed record XmlSchemaValidationError(string Message, string? Location); diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs new file mode 100644 index 00000000..06a72219 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Common.Xml; + +public sealed class XmlSchemaValidationException : Exception +{ + public XmlSchemaValidationException(string documentName, IReadOnlyList errors) + : base($"XML schema validation failed for '{documentName}'.") + { + DocumentName = documentName; + Errors = errors ?? Array.Empty(); + } + + public string DocumentName { get; } + + public IReadOnlyList Errors { get; } +} diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs new file mode 100644 index 00000000..f8de8437 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Xml.Linq; +using System.Xml.Schema; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.Source.Common.Xml; + +public sealed class XmlSchemaValidator : IXmlSchemaValidator +{ + private readonly ILogger _logger; + + public XmlSchemaValidator(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void Validate(XDocument document, XmlSchemaSet schemaSet, string documentName) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(schemaSet); + ArgumentException.ThrowIfNullOrWhiteSpace(documentName); + + var errors = new List(); + + void Handler(object? sender, ValidationEventArgs args) + { + if (args is null) + { + return; + } + + var location = FormatLocation(args.Exception); + errors.Add(new XmlSchemaValidationError(args.Message, location)); + } + + try + { + document.Validate(schemaSet, Handler, addSchemaInfo: true); + } + catch (System.Xml.Schema.XmlSchemaValidationException ex) + { + var location = FormatLocation(ex); + errors.Add(new XmlSchemaValidationError(ex.Message, location)); + } + + if (errors.Count > 0) + { + var exception = new XmlSchemaValidationException(documentName, errors); + _logger.LogError(exception, "XML schema validation failed for {DocumentName}", documentName); + throw exception; + } + + _logger.LogDebug("XML schema validation succeeded for {DocumentName}", documentName); + } + + private static string? FormatLocation(System.Xml.Schema.XmlSchemaException? exception) + { + if (exception is null) + { + return null; + } + + if (exception.LineNumber <= 0) + { + return null; + } + + return $"line {exception.LineNumber}, position {exception.LinePosition}"; + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Class1.cs b/src/StellaOps.Feedser.Source.Cve/Class1.cs new file mode 100644 index 00000000..1f26fab1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Cve; + +public sealed class CveConnectorPlugin : IConnectorPlugin +{ + public string Name => "cve"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj b/src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Class1.cs b/src/StellaOps.Feedser.Source.Distro.Debian/Class1.cs new file mode 100644 index 00000000..acbcc9b4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Debian; + +public sealed class DistroDebianConnectorPlugin : IConnectorPlugin +{ + public string Name => "distro-debian"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj b/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json new file mode 100644 index 00000000..41c69f8d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json @@ -0,0 +1,95 @@ +{ + "document": { + "aggregate_severity": { + "text": "Important" + }, + "lang": "en", + "notes": [ + { + "category": "summary", + "text": "An update fixes a critical kernel issue." + } + ], + "references": [ + { + "category": "self", + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0001" + } + ], + "title": "Red Hat Security Advisory: Example kernel update", + "tracking": { + "id": "RHSA-2025:0001", + "initial_release_date": "2025-10-02T00:00:00+00:00", + "current_release_date": "2025-10-03T00:00:00+00:00" + } + }, + "product_tree": { + "branches": [ + { + "category": "product_family", + "branches": [ + { + "category": "product_name", + "product": { + "name": "Red Hat Enterprise Linux 8", + "product_id": "8Base-RHEL-8", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:8" + } + } + } + ] + }, + { + "category": "product_release", + "branches": [ + { + "category": "product_version", + "product": { + "name": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "product_id": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "product_identification_helper": { + "purl": "pkg:rpm/redhat/kernel@4.18.0-513.5.1.el8?arch=x86_64" + } + } + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0001", + "references": [ + { + "category": "external", + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0001" + } + ], + "scores": [ + { + "cvss_v3": { + "baseScore": 9.8, + "baseSeverity": "CRITICAL", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + } + ], + "product_status": { + "fixed": [ + "8Base-RHEL-8:kernel-0:4.18.0-513.5.1.el8.x86_64" + ], + "first_fixed": [ + "8Base-RHEL-8:kernel-0:4.18.0-513.5.1.el8.x86_64" + ], + "known_affected": [ + "8Base-RHEL-8", + "8Base-RHEL-8:kernel-0:4.18.0-500.1.0.el8.x86_64" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json new file mode 100644 index 00000000..210c6cab --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json @@ -0,0 +1,82 @@ +{ + "document": { + "aggregate_severity": { + "text": "Moderate" + }, + "lang": "en", + "notes": [ + { + "category": "summary", + "text": "Second advisory covering unaffected packages." + } + ], + "references": [ + { + "category": "self", + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0002" + } + ], + "title": "Red Hat Security Advisory: Follow-up kernel status", + "tracking": { + "id": "RHSA-2025:0002", + "initial_release_date": "2025-10-05T12:00:00+00:00", + "current_release_date": "2025-10-05T12:00:00+00:00" + } + }, + "product_tree": { + "branches": [ + { + "category": "product_family", + "branches": [ + { + "category": "product_name", + "product": { + "name": "Red Hat Enterprise Linux 9", + "product_id": "9Base-RHEL-9", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:9" + } + } + } + ] + }, + { + "category": "product_release", + "branches": [ + { + "category": "product_version", + "product": { + "name": "kernel-0:5.14.0-400.el9.x86_64", + "product_id": "kernel-0:5.14.0-400.el9.x86_64", + "product_identification_helper": { + "purl": "pkg:rpm/redhat/kernel@5.14.0-400.el9?arch=x86_64" + } + } + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0002", + "references": [ + { + "category": "external", + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0002" + } + ], + "product_status": { + "known_not_affected": [ + "9Base-RHEL-9", + "9Base-RHEL-9:kernel-0:5.14.0-400.el9.x86_64" + ], + "under_investigation": [ + "9Base-RHEL-9:kernel-0:5.14.0-401.el9.x86_64" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json new file mode 100644 index 00000000..b32f4284 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json @@ -0,0 +1,93 @@ +{ + "document": { + "aggregate_severity": { + "text": "Important" + }, + "lang": "en", + "notes": [ + { + "category": "summary", + "text": "Advisory with mixed reference sources to verify dedupe ordering." + } + ], + "references": [ + { + "category": "self", + "summary": "Primary advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + { + "category": "self", + "summary": "", + "url": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + { + "category": "mitigation", + "summary": "Knowledge base guidance", + "url": "https://access.redhat.com/solutions/999999" + } + ], + "title": "Red Hat Security Advisory: Reference dedupe validation", + "tracking": { + "id": "RHSA-2025:0003", + "initial_release_date": "2025-10-06T09:00:00+00:00", + "current_release_date": "2025-10-06T09:00:00+00:00" + } + }, + "product_tree": { + "branches": [ + { + "category": "product_family", + "branches": [ + { + "category": "product_name", + "product": { + "name": "Red Hat Enterprise Linux 9", + "product_id": "9Base-RHEL-9", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:9" + } + } + } + ] + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0003", + "references": [ + { + "category": "external", + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + }, + { + "category": "external", + "summary": "", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + }, + { + "category": "exploit", + "summary": "Exploit tracking", + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" + } + ], + "scores": [ + { + "cvss_v3": { + "baseScore": 7.5, + "baseSeverity": "HIGH", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + "version": "3.1" + } + } + ], + "product_status": { + "known_affected": [ + "9Base-RHEL-9" + ] + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json new file mode 100644 index 00000000..bb3bfe24 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json @@ -0,0 +1,118 @@ +{ + "advisoryKey": "RHSA-2025:0001", + "affectedPackages": [ + { + "identifier": "cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 8", + "provenance": [ + { + "kind": "oval", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "8Base-RHEL-8" + } + ], + "statuses": [ + { + "provenance": { + "kind": "oval", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "8Base-RHEL-8" + }, + "status": "known_affected" + } + ], + "type": "cpe", + "versionRanges": [] + }, + { + "identifier": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "platform": "Red Hat Enterprise Linux 8", + "provenance": [ + { + "kind": "package.nevra", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "kernel-0:4.18.0-513.5.1.el8.x86_64" + } + ], + "statuses": [], + "type": "rpm", + "versionRanges": [ + { + "fixedVersion": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "introducedVersion": null, + "lastAffectedVersion": null, + "provenance": { + "kind": "package.nevra", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "kernel-0:4.18.0-513.5.1.el8.x86_64" + }, + "rangeExpression": null, + "rangeKind": "nevra" + } + ] + } + ], + "aliases": [ + "CVE-2025-0001", + "RHSA-2025:0001" + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "kind": "cvss", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "CVE-2025-0001" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-03T00:00:00+00:00", + "provenance": [ + { + "kind": "advisory", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "RHSA-2025:0001" + } + ], + "published": "2025-10-02T00:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "kind": "reference", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "https://access.redhat.com/errata/RHSA-2025:0001" + }, + "sourceTag": null, + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0001" + }, + { + "kind": "external", + "provenance": { + "kind": "reference", + "recordedAt": "2025-10-05T00:00:00+00:00", + "source": "redhat", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0001" + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0001" + } + ], + "severity": "high", + "summary": "An update fixes a critical kernel issue.", + "title": "Red Hat Security Advisory: Example kernel update" +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json new file mode 100644 index 00000000..c8cca679 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json @@ -0,0 +1,8 @@ +[ + { + "RHSA": "RHSA-2025:0001", + "severity": "important", + "released_on": "2025-10-03T00:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json new file mode 100644 index 00000000..1aaeebdb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json @@ -0,0 +1,8 @@ +[ + { + "RHSA": "RHSA-2025:0001", + "severity": "important", + "released_on": "2025-10-03T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json new file mode 100644 index 00000000..b90bdec8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json @@ -0,0 +1,8 @@ +[ + { + "RHSA": "RHSA-2025:0002", + "severity": "moderate", + "released_on": "2025-10-05T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json new file mode 100644 index 00000000..d55083cc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json @@ -0,0 +1,8 @@ +[ + { + "RHSA": "RHSA-2025:0003", + "severity": "important", + "released_on": "2025-10-06T09:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0003.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs new file mode 100644 index 00000000..325b8846 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs @@ -0,0 +1,114 @@ +using System; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.RedHat; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Tests; + +[Collection("mongo-fixture")] +public sealed class RedHatConnectorHarnessTests : IAsyncLifetime +{ + private readonly ConnectorTestHarness _harness; + + public RedHatConnectorHarnessTests(MongoIntegrationFixture fixture) + { + _harness = new ConnectorTestHarness(fixture, new DateTimeOffset(2025, 10, 5, 0, 0, 0, TimeSpan.Zero), RedHatOptions.HttpClientName); + } + + [Fact] + public async Task FetchParseMap_WithHarness_ProducesCanonicalAdvisory() + { + await _harness.ResetAsync(); + + var options = new RedHatOptions + { + BaseEndpoint = new Uri("https://access.redhat.com/hydra/rest/securitydata"), + PageSize = 1, + MaxPagesPerFetch = 1, + MaxAdvisoriesPerFetch = 5, + InitialBackfill = TimeSpan.FromDays(1), + Overlap = TimeSpan.Zero, + FetchTimeout = TimeSpan.FromSeconds(30), + UserAgent = "StellaOps.Tests.RedHatHarness/1.0", + }; + + var handler = _harness.Handler; + var timeProvider = _harness.TimeProvider; + + var summaryUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-04&per_page=1&page=1"); + var detailUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json"); + + handler.AddJsonResponse(summaryUri, ReadFixture("summary-page1.json")); + handler.AddJsonResponse(detailUri, ReadFixture("csaf-rhsa-2025-0001.json")); + + await _harness.EnsureServiceProviderAsync(services => + { + services.AddRedHatConnector(opts => + { + opts.BaseEndpoint = options.BaseEndpoint; + opts.PageSize = options.PageSize; + opts.MaxPagesPerFetch = options.MaxPagesPerFetch; + opts.MaxAdvisoriesPerFetch = options.MaxAdvisoriesPerFetch; + opts.InitialBackfill = options.InitialBackfill; + opts.Overlap = options.Overlap; + opts.FetchTimeout = options.FetchTimeout; + opts.UserAgent = options.UserAgent; + }); + }); + + var provider = _harness.ServiceProvider; + + var stateRepository = provider.GetRequiredService(); + await stateRepository.UpsertAsync( + new SourceStateRecord( + RedHatConnectorPlugin.SourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument(), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: timeProvider.GetUtcNow(), + LastFailureReason: null), + CancellationToken.None); + + var connector = new RedHatConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + Assert.Single(advisories); + var advisory = advisories.Single(); + Assert.Equal("RHSA-2025:0001", advisory.AdvisoryKey); + Assert.Equal("high", advisory.Severity); + Assert.Contains(advisory.Aliases, alias => alias == "CVE-2025-0001"); + Assert.Empty(advisory.Provenance.Where(p => p.Source == "redhat" && p.Kind == "fetch")); + + var state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings) && pendingMappings.AsBsonArray.Count == 0); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => _harness.ResetAsync(); + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "RedHat", "Fixtures", filename); + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs new file mode 100644 index 00000000..dcd6351a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs @@ -0,0 +1,449 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Distro.RedHat; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Testing; +using StellaOps.Plugin; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Tests; + +[Collection("mongo-fixture")] +public sealed class RedHatConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly DateTimeOffset _initialNow; + private readonly CannedHttpMessageHandler _handler; + private readonly ITestOutputHelper _output; + private ServiceProvider? _serviceProvider; + + public RedHatConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _initialNow = new DateTimeOffset(2025, 10, 5, 0, 0, 0, TimeSpan.Zero); + _timeProvider = new FakeTimeProvider(_initialNow); + _handler = new CannedHttpMessageHandler(); + _output = output; + } + + [Fact] + public async Task FetchParseMap_ProducesCanonicalAdvisory() + { + await ResetDatabaseAsync(); + + var options = new RedHatOptions + { + BaseEndpoint = new Uri("https://access.redhat.com/hydra/rest/securitydata"), + PageSize = 1, + MaxPagesPerFetch = 1, + MaxAdvisoriesPerFetch = 25, + InitialBackfill = TimeSpan.FromDays(1), + Overlap = TimeSpan.Zero, + FetchTimeout = TimeSpan.FromSeconds(30), + UserAgent = "StellaOps.Tests.RedHat/1.0", + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + var configuredOptions = provider.GetRequiredService>().Value; + Assert.Equal(1, configuredOptions.PageSize); + + var summaryUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-04&per_page=1&page=1"); + var detailUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json"); + + _output.WriteLine($"Registering summary URI: {summaryUri}"); + _handler.AddJsonResponse(summaryUri, ReadFixture("summary-page1.json")); + _handler.AddJsonResponse(detailUri, ReadFixture("csaf-rhsa-2025-0001.json")); + + var stateRepository = provider.GetRequiredService(); + await stateRepository.UpsertAsync( + new SourceStateRecord( + RedHatConnectorPlugin.SourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument(), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: _timeProvider.GetUtcNow(), + LastFailureReason: null), + CancellationToken.None); + + var connector = new RedHatConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var advisory = advisories.Single(a => string.Equals(a.AdvisoryKey, "RHSA-2025:0001", StringComparison.Ordinal)); + Assert.Equal("red hat security advisory: example kernel update", advisory.Title.ToLowerInvariant()); + Assert.Contains("RHSA-2025:0001", advisory.Aliases); + Assert.Contains("CVE-2025-0001", advisory.Aliases); + Assert.Equal("high", advisory.Severity); + Assert.Equal("en", advisory.Language); + + var rpmPackage = advisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Rpm); + _output.WriteLine($"RPM statuses count: {rpmPackage.Statuses.Length}"); + _output.WriteLine($"RPM ranges count: {rpmPackage.VersionRanges.Length}"); + foreach (var range in rpmPackage.VersionRanges) + { + _output.WriteLine($"Range fixed={range.FixedVersion}, last={range.LastAffectedVersion}, expr={range.RangeExpression}"); + } + Assert.Equal("kernel-0:4.18.0-513.5.1.el8.x86_64", rpmPackage.Identifier); + var fixedRange = Assert.Single( + rpmPackage.VersionRanges, + range => string.Equals(range.FixedVersion, "kernel-0:4.18.0-513.5.1.el8.x86_64", StringComparison.Ordinal)); + Assert.Equal("kernel-0:4.18.0-500.1.0.el8.x86_64", fixedRange.LastAffectedVersion); + + var cpePackage = advisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Cpe); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", cpePackage.Identifier); + + Assert.Contains(advisory.References, reference => reference.Url == "https://access.redhat.com/errata/RHSA-2025:0001"); + Assert.Contains(advisory.References, reference => reference.Url == "https://www.cve.org/CVERecord?id=CVE-2025-0001"); + + var snapshot = SnapshotSerializer.ToSnapshot(advisory); + _output.WriteLine("-- RHSA-2025:0001 snapshot --\n" + snapshot); + var snapshotPath = Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "RedHat", "Fixtures", "rhsa-2025-0001.snapshot.json"); + var expectedSnapshot = File.ReadAllText(snapshotPath); + Assert.Equal(expectedSnapshot, snapshot); + + var state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings) && pendingMappings.AsBsonArray.Count == 0); + + const string fetchKind = "source:redhat:fetch"; + const string parseKind = "source:redhat:parse"; + const string mapKind = "source:redhat:map"; + + var schedulerOptions = provider.GetRequiredService>().Value; + Assert.True(schedulerOptions.Definitions.TryGetValue(fetchKind, out var fetchDefinition)); + Assert.True(schedulerOptions.Definitions.TryGetValue(parseKind, out var parseDefinition)); + Assert.True(schedulerOptions.Definitions.TryGetValue(mapKind, out var mapDefinition)); + + Assert.Equal("RedHatFetchJob", fetchDefinition.JobType.Name); + Assert.Equal(TimeSpan.FromMinutes(12), fetchDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(6), fetchDefinition.LeaseDuration); + Assert.Equal("0,15,30,45 * * * *", fetchDefinition.CronExpression); + Assert.True(fetchDefinition.Enabled); + + Assert.Equal("RedHatParseJob", parseDefinition.JobType.Name); + Assert.Equal(TimeSpan.FromMinutes(15), parseDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(6), parseDefinition.LeaseDuration); + Assert.Equal("5,20,35,50 * * * *", parseDefinition.CronExpression); + Assert.True(parseDefinition.Enabled); + + Assert.Equal("RedHatMapJob", mapDefinition.JobType.Name); + Assert.Equal(TimeSpan.FromMinutes(20), mapDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(6), mapDefinition.LeaseDuration); + Assert.Equal("10,25,40,55 * * * *", mapDefinition.CronExpression); + Assert.True(mapDefinition.Enabled); + + var summaryUriRepeat = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-03&per_page=10&page=1"); + var summaryUriSecondPage = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-03&per_page=10&page=2"); + var detailUri2 = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json"); + + _output.WriteLine($"Registering repeat summary URI: {summaryUriRepeat}"); + _output.WriteLine($"Registering second page summary URI: {summaryUriSecondPage}"); + _handler.AddJsonResponse(summaryUriRepeat, ReadFixture("summary-page1-repeat.json")); + _handler.AddJsonResponse(summaryUriSecondPage, ReadFixture("summary-page2.json")); + _handler.AddJsonResponse(detailUri2, ReadFixture("csaf-rhsa-2025-0002.json")); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var secondAdvisory = advisories.Single(a => string.Equals(a.AdvisoryKey, "RHSA-2025:0002", StringComparison.Ordinal)); + var rpm2 = secondAdvisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Rpm); + Assert.Equal("kernel-0:5.14.0-400.el9.x86_64", rpm2.Identifier); + const string knownNotAffected = "known_not_affected"; + const string underInvestigation = "under_investigation"; + + Assert.DoesNotContain(rpm2.VersionRanges, range => string.Equals(range.RangeExpression, knownNotAffected, StringComparison.Ordinal)); + Assert.DoesNotContain(rpm2.VersionRanges, range => string.Equals(range.RangeExpression, underInvestigation, StringComparison.Ordinal)); + Assert.Contains(rpm2.Statuses, status => status.Status == knownNotAffected); + Assert.Contains(rpm2.Statuses, status => status.Status == underInvestigation); + + var cpe2 = secondAdvisory.AffectedPackages.Single(pkg => pkg.Type == AffectedPackageTypes.Cpe); + Assert.Equal("cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", cpe2.Identifier); + Assert.Empty(cpe2.VersionRanges); + Assert.Contains(cpe2.Statuses, status => status.Status == knownNotAffected); + + state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out pendingMappings) && pendingMappings.AsBsonArray.Count == 0); + } + + [Fact] + public async Task Resume_CompletesPendingDocumentsAfterRestart() + { + await ResetDatabaseAsync(); + + var options = new RedHatOptions + { + BaseEndpoint = new Uri("https://access.redhat.com/hydra/rest/securitydata"), + PageSize = 1, + MaxPagesPerFetch = 1, + MaxAdvisoriesPerFetch = 25, + InitialBackfill = TimeSpan.FromDays(1), + Overlap = TimeSpan.Zero, + FetchTimeout = TimeSpan.FromSeconds(30), + UserAgent = "StellaOps.Tests.RedHat/1.0", + }; + + var summaryUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-04&per_page=1&page=1"); + var detailUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json"); + + var fetchHandler = new CannedHttpMessageHandler(); + fetchHandler.AddJsonResponse(summaryUri, ReadFixture("summary-page1.json")); + fetchHandler.AddJsonResponse(detailUri, ReadFixture("csaf-rhsa-2025-0001.json")); + + Guid[] pendingDocumentIds; + await using (var fetchProvider = await CreateServiceProviderAsync(options, fetchHandler)) + { + var stateRepository = fetchProvider.GetRequiredService(); + await stateRepository.UpsertAsync( + new SourceStateRecord( + RedHatConnectorPlugin.SourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument(), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: _timeProvider.GetUtcNow(), + LastFailureReason: null), + CancellationToken.None); + + var connector = new RedHatConnectorPlugin().Create(fetchProvider); + await connector.FetchAsync(fetchProvider, CancellationToken.None); + + var state = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocs = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.NotEmpty(pendingDocs); + pendingDocumentIds = pendingDocs.Select(value => Guid.Parse(value.AsString)).ToArray(); + } + + var resumeHandler = new CannedHttpMessageHandler(); + await using (var resumeProvider = await CreateServiceProviderAsync(options, resumeHandler)) + { + var resumeConnector = new RedHatConnectorPlugin().Create(resumeProvider); + + await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None); + await resumeConnector.MapAsync(resumeProvider, CancellationToken.None); + + var documentStore = resumeProvider.GetRequiredService(); + foreach (var documentId in pendingDocumentIds) + { + var document = await documentStore.FindAsync(documentId, CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + } + + var advisoryStore = resumeProvider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.NotEmpty(advisories); + + var stateRepository = resumeProvider.GetRequiredService(); + var finalState = await stateRepository.TryGetAsync(RedHatConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(finalState); + var finalPendingDocs = finalState!.Cursor.TryGetValue("pendingDocuments", out var docsValue) ? docsValue.AsBsonArray : new BsonArray(); + Assert.Empty(finalPendingDocs); + var finalPendingMappings = finalState.Cursor.TryGetValue("pendingMappings", out var mappingsValue) ? mappingsValue.AsBsonArray : new BsonArray(); + Assert.Empty(finalPendingMappings); + } + } + + [Fact] + public async Task MapAsync_DeduplicatesReferencesAndOrdersDeterministically() + { + await ResetDatabaseAsync(); + + var options = new RedHatOptions + { + BaseEndpoint = new Uri("https://access.redhat.com/hydra/rest/securitydata"), + PageSize = 1, + MaxPagesPerFetch = 1, + MaxAdvisoriesPerFetch = 10, + InitialBackfill = TimeSpan.FromDays(7), + Overlap = TimeSpan.Zero, + FetchTimeout = TimeSpan.FromSeconds(30), + UserAgent = "StellaOps.Tests.RedHat/1.0", + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + var summaryUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf.json?after=2025-10-05&per_page=1&page=1"); + var detailUri = new Uri("https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0003.json"); + + _handler.AddJsonResponse(summaryUri, ReadFixture("summary-page3.json")); + _handler.AddJsonResponse(detailUri, ReadFixture("csaf-rhsa-2025-0003.json")); + + var stateRepository = provider.GetRequiredService(); + await stateRepository.UpsertAsync( + new SourceStateRecord( + RedHatConnectorPlugin.SourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument(), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: _timeProvider.GetUtcNow(), + LastFailureReason: null), + CancellationToken.None); + + var connector = new RedHatConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisory = (await advisoryStore.GetRecentAsync(10, CancellationToken.None)) + .Single(a => string.Equals(a.AdvisoryKey, "RHSA-2025:0003", StringComparison.Ordinal)); + + var references = advisory.References.ToArray(); + Assert.Equal(4, references.Length); + + Assert.Equal("exploit", references[0].Kind); + Assert.Equal("https://bugzilla.redhat.com/show_bug.cgi?id=2222222", references[0].Url); + + Assert.Equal("external", references[1].Kind); + Assert.Equal("https://www.cve.org/CVERecord?id=CVE-2025-0003", references[1].Url); + Assert.Equal("CVE record", references[1].Summary); + + Assert.Equal("mitigation", references[2].Kind); + Assert.Equal("https://access.redhat.com/solutions/999999", references[2].Url); + Assert.Equal("Knowledge base guidance", references[2].Summary); + + Assert.Equal("self", references[3].Kind); + Assert.Equal("https://access.redhat.com/errata/RHSA-2025:0003", references[3].Url); + Assert.Equal("Primary advisory", references[3].Summary); + } + + private async Task EnsureServiceProviderAsync(RedHatOptions options) + { + if (_serviceProvider is not null) + { + return; + } + + _serviceProvider = await CreateServiceProviderAsync(options, _handler); + } + + private async Task CreateServiceProviderAsync(RedHatOptions options, CannedHttpMessageHandler handler) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(storageOptions => + { + storageOptions.ConnectionString = _fixture.Runner.ConnectionString; + storageOptions.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + storageOptions.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddRedHatConnector(opts => + { + opts.BaseEndpoint = options.BaseEndpoint; + opts.PageSize = options.PageSize; + opts.MaxPagesPerFetch = options.MaxPagesPerFetch; + opts.MaxAdvisoriesPerFetch = options.MaxAdvisoriesPerFetch; + opts.InitialBackfill = options.InitialBackfill; + opts.Overlap = options.Overlap; + opts.FetchTimeout = options.FetchTimeout; + opts.UserAgent = options.UserAgent; + }); + + services.Configure(RedHatOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private Task ResetDatabaseAsync() + { + return ResetDatabaseInternalAsync(); + } + + private async Task ResetDatabaseInternalAsync() + { + if (_serviceProvider is not null) + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider.Dispose(); + } + + _serviceProvider = null; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + _timeProvider.SetUtcNow(_initialNow); + } + + private static string ReadFixture(string name) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "RedHat", "Fixtures", name); + return File.ReadAllText(path); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await ResetDatabaseInternalAsync(); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj new file mode 100644 index 00000000..695afed4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md b/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md new file mode 100644 index 00000000..50dfbcef --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +Red Hat distro connector (Security Data API and OVAL) providing authoritative OS package ranges (RPM NEVRA) and RHSA metadata; overrides generic registry ranges during merge. +## Scope +- Fetch Security Data JSON (for example CVRF) via Hydra; window by last_modified or after cursor; optionally ingest OVAL definitions. +- Validate payloads; parse advisories, CVEs, affected packages; materialize NEVRA and CPE records. +- Map to canonical advisories with affected Type=rpm/cpe, fixedBy NEVRA, RHSA aliasing; persist provenance indicating oval/package.nevra. +## Participants +- Source.Common (HTTP, throttling, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, source_state). +- Models (canonical Affected with NEVRA). +- Core/WebService (jobs: source:redhat:fetch|parse|map) already registered. +- Merge engine to enforce distro precedence (OVAL or PSIRT greater than NVD). +## Interfaces & contracts +- Aliases: RHSA-YYYY:NNNN, CVE ids; references include RHSA pages, errata, OVAL links. +- Affected: rpm (Identifier=NEVRA key) and cpe entries; versions include introduced/fixed/fixedBy; platforms mark RHEL streams. +- Provenance: kind="oval" or "package.nevra" as applicable; value=definition id or package. +## In/Out of scope +In: authoritative rpm ranges, RHSA mapping, OVAL interpretation, watermarking. +Out: building RPM artifacts; cross-distro reconciliation beyond Red Hat. +## Observability & security expectations +- Metrics: redhat.fetch.items, redhat.oval.defs, redhat.parse.fail, redhat.map.affected_rpm. +- Logs: cursor bounds, advisory ids, NEVRA counts; allowlist Red Hat endpoints. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Distro.RedHat.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs new file mode 100644 index 00000000..aed285f3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs @@ -0,0 +1,97 @@ +namespace StellaOps.Feedser.Source.Distro.RedHat.Configuration; + +public sealed class RedHatOptions +{ + /// + /// Name of the HttpClient registered for Red Hat Hydra requests. + /// + public const string HttpClientName = "redhat-hydra"; + + /// + /// Base API endpoint for Hydra security data requests. + /// + public Uri BaseEndpoint { get; set; } = new("https://access.redhat.com/hydra/rest/securitydata"); + + /// + /// Relative path for the advisory listing endpoint (returns summary rows with resource_url values). + /// + public string SummaryPath { get; set; } = "csaf.json"; + + /// + /// Number of summary rows requested per page when scanning for new advisories. + /// + public int PageSize { get; set; } = 200; + + /// + /// Maximum number of summary pages to inspect within one fetch invocation. + /// + public int MaxPagesPerFetch { get; set; } = 5; + + /// + /// Upper bound on individual advisories fetched per invocation (guards against unbounded catch-up floods). + /// + public int MaxAdvisoriesPerFetch { get; set; } = 800; + + /// + /// Initial look-back window applied when no watermark exists (Red Hat publishes extensive history; we default to 30 days). + /// + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + /// + /// Optional overlap period re-scanned on each run to pick up late-published advisories. + /// + public TimeSpan Overlap { get; set; } = TimeSpan.FromDays(1); + + /// + /// Timeout applied to individual Hydra document fetches. + /// + public TimeSpan FetchTimeout { get; set; } = TimeSpan.FromSeconds(60); + + /// + /// Custom user-agent presented to Red Hat endpoints (kept short to satisfy Jetty header limits). + /// + public string UserAgent { get; set; } = "StellaOps.Feedser.RedHat/1.0"; + + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("Red Hat Hydra base endpoint must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(SummaryPath)) + { + throw new InvalidOperationException("Red Hat Hydra summary path must be configured."); + } + + if (PageSize <= 0) + { + throw new InvalidOperationException("Red Hat Hydra page size must be positive."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("Red Hat Hydra max pages per fetch must be positive."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException("Red Hat Hydra max advisories per fetch must be positive."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Red Hat Hydra initial backfill must be positive."); + } + + if (Overlap < TimeSpan.Zero) + { + throw new InvalidOperationException("Red Hat Hydra overlap cannot be negative."); + } + + if (FetchTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Red Hat Hydra fetch timeout must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs new file mode 100644 index 00000000..942f2597 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Internal.Models; + +internal sealed class RedHatCsafEnvelope +{ + [JsonPropertyName("document")] + public RedHatDocumentSection? Document { get; init; } + + [JsonPropertyName("product_tree")] + public RedHatProductTree? ProductTree { get; init; } + + [JsonPropertyName("vulnerabilities")] + public IReadOnlyList? Vulnerabilities { get; init; } +} + +internal sealed class RedHatDocumentSection +{ + [JsonPropertyName("aggregate_severity")] + public RedHatAggregateSeverity? AggregateSeverity { get; init; } + + [JsonPropertyName("lang")] + public string? Lang { get; init; } + + [JsonPropertyName("notes")] + public IReadOnlyList? Notes { get; init; } + + [JsonPropertyName("references")] + public IReadOnlyList? References { get; init; } + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("tracking")] + public RedHatTracking? Tracking { get; init; } +} + +internal sealed class RedHatAggregateSeverity +{ + [JsonPropertyName("text")] + public string? Text { get; init; } +} + +internal sealed class RedHatDocumentNote +{ + [JsonPropertyName("category")] + public string? Category { get; init; } + + [JsonPropertyName("text")] + public string? Text { get; init; } + + public bool CategoryEquals(string value) + => !string.IsNullOrWhiteSpace(Category) + && string.Equals(Category, value, StringComparison.OrdinalIgnoreCase); +} + +internal sealed class RedHatTracking +{ + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("initial_release_date")] + public DateTimeOffset? InitialReleaseDate { get; init; } + + [JsonPropertyName("current_release_date")] + public DateTimeOffset? CurrentReleaseDate { get; init; } +} + +internal sealed class RedHatReference +{ + [JsonPropertyName("category")] + public string? Category { get; init; } + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("url")] + public string? Url { get; init; } +} + +internal sealed class RedHatProductTree +{ + [JsonPropertyName("branches")] + public IReadOnlyList? Branches { get; init; } +} + +internal sealed class RedHatProductBranch +{ + [JsonPropertyName("category")] + public string? Category { get; init; } + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("product")] + public RedHatProductNodeInfo? Product { get; init; } + + [JsonPropertyName("branches")] + public IReadOnlyList? Branches { get; init; } +} + +internal sealed class RedHatProductNodeInfo +{ + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("product_id")] + public string? ProductId { get; init; } + + [JsonPropertyName("product_identification_helper")] + public RedHatProductIdentificationHelper? ProductIdentificationHelper { get; init; } +} + +internal sealed class RedHatProductIdentificationHelper +{ + [JsonPropertyName("cpe")] + public string? Cpe { get; init; } + + [JsonPropertyName("purl")] + public string? Purl { get; init; } +} + +internal sealed class RedHatVulnerability +{ + [JsonPropertyName("cve")] + public string? Cve { get; init; } + + [JsonPropertyName("references")] + public IReadOnlyList? References { get; init; } + + [JsonPropertyName("scores")] + public IReadOnlyList? Scores { get; init; } + + [JsonPropertyName("product_status")] + public RedHatProductStatus? ProductStatus { get; init; } +} + +internal sealed class RedHatVulnerabilityScore +{ + [JsonPropertyName("cvss_v3")] + public RedHatCvssV3? CvssV3 { get; init; } +} + +internal sealed class RedHatCvssV3 +{ + [JsonPropertyName("baseScore")] + public double? BaseScore { get; init; } + + [JsonPropertyName("baseSeverity")] + public string? BaseSeverity { get; init; } + + [JsonPropertyName("vectorString")] + public string? VectorString { get; init; } + + [JsonPropertyName("version")] + public string? Version { get; init; } +} + +internal sealed class RedHatProductStatus +{ + [JsonPropertyName("fixed")] + public IReadOnlyList? Fixed { get; init; } + + [JsonPropertyName("first_fixed")] + public IReadOnlyList? FirstFixed { get; init; } + + [JsonPropertyName("known_affected")] + public IReadOnlyList? KnownAffected { get; init; } + + [JsonPropertyName("known_not_affected")] + public IReadOnlyList? KnownNotAffected { get; init; } + + [JsonPropertyName("under_investigation")] + public IReadOnlyList? UnderInvestigation { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs new file mode 100644 index 00000000..b55763e0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs @@ -0,0 +1,254 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; + +internal sealed record RedHatCursor( + DateTimeOffset? LastReleasedOn, + IReadOnlyCollection ProcessedAdvisoryIds, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + private static readonly IReadOnlyCollection EmptyStringList = Array.Empty(); + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static RedHatCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList, EmptyCache); + + public static RedHatCursor FromBsonDocument(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastReleased = null; + if (document.TryGetValue("lastReleasedOn", out var lastReleasedValue)) + { + lastReleased = ReadDateTimeOffset(lastReleasedValue); + } + + var processed = ReadStringSet(document, "processedAdvisories"); + var pendingDocuments = ReadGuidSet(document, "pendingDocuments"); + var pendingMappings = ReadGuidSet(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new RedHatCursor(lastReleased, processed, pendingDocuments, pendingMappings, fetchCache); + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (LastReleasedOn.HasValue) + { + document["lastReleasedOn"] = LastReleasedOn.Value.UtcDateTime; + } + + document["processedAdvisories"] = new BsonArray(ProcessedAdvisoryIds); + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); + + var cacheArray = new BsonArray(); + foreach (var (key, metadata) in FetchCache) + { + var cacheDoc = new BsonDocument + { + ["uri"] = key + }; + + if (!string.IsNullOrWhiteSpace(metadata.ETag)) + { + cacheDoc["etag"] = metadata.ETag; + } + + if (metadata.LastModified.HasValue) + { + cacheDoc["lastModified"] = metadata.LastModified.Value.UtcDateTime; + } + + cacheArray.Add(cacheDoc); + } + + document["fetchCache"] = cacheArray; + return document; + } + + public RedHatCursor WithLastReleased(DateTimeOffset? releasedOn, IEnumerable advisoryIds) + { + var normalizedIds = advisoryIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? Array.Empty(); + + return this with + { + LastReleasedOn = releasedOn, + ProcessedAdvisoryIds = normalizedIds + }; + } + + public RedHatCursor AddProcessedAdvisories(IEnumerable advisoryIds) + { + if (advisoryIds is null) + { + return this; + } + + var set = new HashSet(ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); + foreach (var id in advisoryIds) + { + if (!string.IsNullOrWhiteSpace(id)) + { + set.Add(id.Trim()); + } + } + + return this with { ProcessedAdvisoryIds = set.ToArray() }; + } + + public RedHatCursor WithPendingDocuments(IEnumerable ids) + { + var list = ids?.Distinct().ToArray() ?? Array.Empty(); + return this with { PendingDocuments = list }; + } + + public RedHatCursor WithPendingMappings(IEnumerable ids) + { + var list = ids?.Distinct().ToArray() ?? Array.Empty(); + return this with { PendingMappings = list }; + } + + public RedHatCursor WithFetchCache(string requestUri, string? etag, DateTimeOffset? lastModified) + { + var cache = new Dictionary(FetchCache, StringComparer.OrdinalIgnoreCase) + { + [requestUri] = new RedHatCachedFetchMetadata(etag, lastModified) + }; + + return this with { FetchCache = cache }; + } + + public RedHatCursor PruneFetchCache(IEnumerable keepUris) + { + if (FetchCache.Count == 0) + { + return this; + } + + var keepSet = new HashSet(keepUris ?? Array.Empty(), StringComparer.OrdinalIgnoreCase); + if (keepSet.Count == 0) + { + return this with { FetchCache = EmptyCache }; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var uri in keepSet) + { + if (FetchCache.TryGetValue(uri, out var metadata)) + { + cache[uri] = metadata; + } + } + + return this with { FetchCache = cache }; + } + + public RedHatCachedFetchMetadata? TryGetFetchCache(string requestUri) + { + if (FetchCache.TryGetValue(requestUri, out var metadata)) + { + return metadata; + } + + return null; + } + + private static IReadOnlyCollection ReadStringSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyStringList; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String) + { + var str = element.AsString.Trim(); + if (!string.IsNullOrWhiteSpace(str)) + { + results.Add(str); + } + } + } + + return results; + } + + private static IReadOnlyCollection ReadGuidSet(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element.BsonType == BsonType.String && Guid.TryParse(element.AsString, out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonArray array || array.Count == 0) + { + return EmptyCache; + } + + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in array.OfType()) + { + if (!element.TryGetValue("uri", out var uriValue) || uriValue.BsonType != BsonType.String) + { + continue; + } + + var uri = uriValue.AsString; + var etag = element.TryGetValue("etag", out var etagValue) && etagValue.BsonType == BsonType.String + ? etagValue.AsString + : null; + DateTimeOffset? lastModified = null; + if (element.TryGetValue("lastModified", out var lastModifiedValue)) + { + lastModified = ReadDateTimeOffset(lastModifiedValue); + } + + results[uri] = new RedHatCachedFetchMetadata(etag, lastModified); + } + + return results; + } + + private static DateTimeOffset? ReadDateTimeOffset(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } +} + +internal sealed record RedHatCachedFetchMetadata(string? ETag, DateTimeOffset? LastModified); diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs new file mode 100644 index 00000000..fdf211c0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs @@ -0,0 +1,718 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Distro.RedHat.Internal.Models; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; + +internal static class RedHatMapper +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNameCaseInsensitive = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + }; + + public static Advisory? Map(string sourceName, DtoRecord dto, DocumentRecord document, JsonDocument payload) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(payload); + + var csaf = JsonSerializer.Deserialize(payload.RootElement.GetRawText(), SerializerOptions); + var documentSection = csaf?.Document; + if (documentSection is null) + { + return null; + } + + var tracking = documentSection.Tracking; + var advisoryKey = NormalizeId(tracking?.Id) + ?? NormalizeId(TryGetMetadata(document, "advisoryId")) + ?? NormalizeId(document.Uri) + ?? dto.DocumentId.ToString(); + + var title = !string.IsNullOrWhiteSpace(documentSection.Title) + ? DescriptionNormalizer.Normalize(new[] { new LocalizedText(documentSection.Title, documentSection.Lang) }).Text + : string.Empty; + if (string.IsNullOrEmpty(title)) + { + title = advisoryKey; + } + + var description = NormalizeSummary(documentSection); + var summary = string.IsNullOrEmpty(description.Text) ? null : description.Text; + var severity = NormalizeSeverity(documentSection.AggregateSeverity?.Text); + var published = tracking?.InitialReleaseDate; + var modified = tracking?.CurrentReleaseDate ?? published; + var language = description.Language; + + var aliases = BuildAliases(advisoryKey, csaf); + var references = BuildReferences(sourceName, dto.ValidatedAt, documentSection, csaf); + var productIndex = RedHatProductIndex.Build(csaf.ProductTree); + var affectedPackages = BuildAffectedPackages(sourceName, dto.ValidatedAt, csaf, productIndex); + var cvssMetrics = BuildCvssMetrics(sourceName, dto.ValidatedAt, advisoryKey, csaf); + + var provenance = new[] + { + new AdvisoryProvenance(sourceName, "advisory", advisoryKey, dto.ValidatedAt), + }; + + return new Advisory( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics, + provenance); + } + + private static IReadOnlyCollection BuildAliases(string advisoryKey, RedHatCsafEnvelope csaf) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + advisoryKey, + }; + + if (csaf.Vulnerabilities is not null) + { + foreach (var vulnerability in csaf.Vulnerabilities) + { + if (!string.IsNullOrWhiteSpace(vulnerability?.Cve)) + { + aliases.Add(vulnerability!.Cve!.Trim()); + } + } + } + + return aliases; + } + + private static NormalizedDescription NormalizeSummary(RedHatDocumentSection documentSection) + { + var summaryNotes = new List(); + var otherNotes = new List(); + + if (documentSection.Notes is not null) + { + foreach (var note in documentSection.Notes) + { + if (note is null || string.IsNullOrWhiteSpace(note.Text)) + { + continue; + } + + var candidate = new LocalizedText(note.Text, documentSection.Lang); + if (note.CategoryEquals("summary")) + { + summaryNotes.Add(candidate); + } + else + { + otherNotes.Add(candidate); + } + } + } + + var combined = summaryNotes.Count > 0 + ? summaryNotes.Concat(otherNotes).ToList() + : otherNotes; + + return DescriptionNormalizer.Normalize(combined); + } + + private static IReadOnlyCollection BuildReferences( + string sourceName, + DateTimeOffset recordedAt, + RedHatDocumentSection? documentSection, + RedHatCsafEnvelope csaf) + { + var references = new List(); + if (documentSection is not null) + { + AppendReferences(sourceName, recordedAt, documentSection.References, references); + } + + if (csaf.Vulnerabilities is not null) + { + foreach (var vulnerability in csaf.Vulnerabilities) + { + AppendReferences(sourceName, recordedAt, vulnerability?.References, references); + } + } + + return NormalizeReferences(references); + } + + private static void AppendReferences(string sourceName, DateTimeOffset recordedAt, IReadOnlyList? items, ICollection references) + { + if (items is null) + { + return; + } + + foreach (var reference in items) + { + if (reference is null || string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + var url = reference.Url.Trim(); + if (!Validation.LooksLikeHttpUrl(url)) + { + continue; + } + + var provenance = new AdvisoryProvenance(sourceName, "reference", url, recordedAt); + references.Add(new AdvisoryReference(url, reference.Category, null, reference.Summary, provenance)); + } + } + + private static IReadOnlyCollection NormalizeReferences(IReadOnlyCollection references) + { + if (references.Count == 0) + { + return Array.Empty(); + } + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var reference in references) + { + if (!map.TryGetValue(reference.Url, out var existing)) + { + map[reference.Url] = reference; + continue; + } + + map[reference.Url] = MergeReferences(existing, reference); + } + + return map.Values + .OrderBy(static r => r.Kind is null ? 1 : 0) + .ThenBy(static r => r.Kind ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static r => r.Url, StringComparer.OrdinalIgnoreCase) + .ThenBy(static r => r.SourceTag ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static AdvisoryReference MergeReferences(AdvisoryReference existing, AdvisoryReference candidate) + { + var kind = existing.Kind ?? candidate.Kind; + var sourceTag = existing.SourceTag ?? candidate.SourceTag; + var summary = ChoosePreferredSummary(existing.Summary, candidate.Summary); + var provenance = existing.Provenance.RecordedAt <= candidate.Provenance.RecordedAt + ? existing.Provenance + : candidate.Provenance; + + if (kind == existing.Kind + && sourceTag == existing.SourceTag + && summary == existing.Summary + && provenance == existing.Provenance) + { + return existing; + } + + if (kind == candidate.Kind + && sourceTag == candidate.SourceTag + && summary == candidate.Summary + && provenance == candidate.Provenance) + { + return candidate; + } + + return new AdvisoryReference(existing.Url, kind, sourceTag, summary, provenance); + } + + private static string? ChoosePreferredSummary(string? left, string? right) + { + var leftValue = string.IsNullOrWhiteSpace(left) ? null : left; + var rightValue = string.IsNullOrWhiteSpace(right) ? null : right; + + if (leftValue is null) + { + return rightValue; + } + + if (rightValue is null) + { + return leftValue; + } + + return leftValue.Length >= rightValue.Length ? leftValue : rightValue; + } + + private static IReadOnlyCollection BuildAffectedPackages( + string sourceName, + DateTimeOffset recordedAt, + RedHatCsafEnvelope csaf, + RedHatProductIndex productIndex) + { + var rpmPackages = new Dictionary(StringComparer.OrdinalIgnoreCase); + var baseProducts = new Dictionary(StringComparer.OrdinalIgnoreCase); + var knownAffectedByBase = BuildKnownAffectedIndex(csaf); + + if (csaf.Vulnerabilities is not null) + { + foreach (var vulnerability in csaf.Vulnerabilities) + { + if (vulnerability?.ProductStatus is null) + { + continue; + } + + RegisterAll(vulnerability.ProductStatus.Fixed, RedHatProductStatuses.Fixed, productIndex, rpmPackages, baseProducts); + RegisterAll(vulnerability.ProductStatus.FirstFixed, RedHatProductStatuses.FirstFixed, productIndex, rpmPackages, baseProducts); + RegisterAll(vulnerability.ProductStatus.KnownAffected, RedHatProductStatuses.KnownAffected, productIndex, rpmPackages, baseProducts); + RegisterAll(vulnerability.ProductStatus.KnownNotAffected, RedHatProductStatuses.KnownNotAffected, productIndex, rpmPackages, baseProducts); + RegisterAll(vulnerability.ProductStatus.UnderInvestigation, RedHatProductStatuses.UnderInvestigation, productIndex, rpmPackages, baseProducts); + } + } + + var affected = new List(rpmPackages.Count + baseProducts.Count); + + foreach (var rpm in rpmPackages.Values) + { + if (rpm.Statuses.Count == 0) + { + continue; + } + + var ranges = new List(); + var statuses = new List(); + var provenance = new AdvisoryProvenance(sourceName, "package.nevra", rpm.ProductId ?? rpm.Nevra, recordedAt); + + var lastKnownAffected = knownAffectedByBase.TryGetValue(rpm.BaseProductId, out var candidate) + ? candidate + : null; + + if (!string.IsNullOrWhiteSpace(lastKnownAffected) + && string.Equals(lastKnownAffected, rpm.Nevra, StringComparison.OrdinalIgnoreCase)) + { + lastKnownAffected = null; + } + + if (rpm.Statuses.Contains(RedHatProductStatuses.Fixed) || rpm.Statuses.Contains(RedHatProductStatuses.FirstFixed)) + { + ranges.Add(new AffectedVersionRange("nevra", null, rpm.Nevra, lastKnownAffected, null, provenance)); + } + + if (!rpm.Statuses.Contains(RedHatProductStatuses.Fixed) + && !rpm.Statuses.Contains(RedHatProductStatuses.FirstFixed) + && rpm.Statuses.Contains(RedHatProductStatuses.KnownAffected)) + { + ranges.Add(new AffectedVersionRange("nevra", null, null, rpm.Nevra, null, provenance)); + } + + if (rpm.Statuses.Contains(RedHatProductStatuses.KnownNotAffected)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.KnownNotAffected, provenance)); + } + + if (rpm.Statuses.Contains(RedHatProductStatuses.UnderInvestigation)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.UnderInvestigation, provenance)); + } + + if (ranges.Count == 0 && statuses.Count == 0) + { + continue; + } + + affected.Add(new AffectedPackage( + AffectedPackageTypes.Rpm, + rpm.Nevra, + rpm.Platform, + ranges, + statuses, + new[] { provenance })); + } + + foreach (var baseEntry in baseProducts.Values) + { + if (baseEntry.Statuses.Count == 0) + { + continue; + } + + var node = baseEntry.Node; + if (string.IsNullOrWhiteSpace(node.Cpe)) + { + continue; + } + + if (!IdentifierNormalizer.TryNormalizeCpe(node.Cpe, out var normalizedCpe)) + { + continue; + } + + var provenance = new AdvisoryProvenance(sourceName, "oval", node.ProductId, recordedAt); + var statuses = new List(); + + if (baseEntry.Statuses.Contains(RedHatProductStatuses.KnownAffected)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.KnownAffected, provenance)); + } + + if (baseEntry.Statuses.Contains(RedHatProductStatuses.KnownNotAffected)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.KnownNotAffected, provenance)); + } + + if (baseEntry.Statuses.Contains(RedHatProductStatuses.UnderInvestigation)) + { + statuses.Add(new AffectedPackageStatus(RedHatProductStatuses.UnderInvestigation, provenance)); + } + + if (statuses.Count == 0) + { + continue; + } + + affected.Add(new AffectedPackage( + AffectedPackageTypes.Cpe, + normalizedCpe!, + node.Name, + Array.Empty(), + statuses, + new[] { provenance })); + } + + return affected; + } + + private static Dictionary BuildKnownAffectedIndex(RedHatCsafEnvelope csaf) + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (csaf.Vulnerabilities is null) + { + return map; + } + + foreach (var vulnerability in csaf.Vulnerabilities) + { + var entries = vulnerability?.ProductStatus?.KnownAffected; + if (entries is null) + { + continue; + } + + foreach (var entry in entries) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + var colonIndex = entry.IndexOf(':'); + if (colonIndex <= 0) + { + continue; + } + + var baseId = entry[..colonIndex].Trim(); + if (string.IsNullOrEmpty(baseId)) + { + continue; + } + + var candidate = NormalizeNevra(entry[(colonIndex + 1)..]); + if (!string.IsNullOrEmpty(candidate)) + { + map[baseId] = candidate; + } + } + } + + return map; + } + + private static void RegisterAll( + IReadOnlyList? entries, + string status, + RedHatProductIndex productIndex, + IDictionary rpmPackages, + IDictionary baseProducts) + { + if (entries is null) + { + return; + } + + foreach (var entry in entries) + { + RegisterProductStatus(entry, status, productIndex, rpmPackages, baseProducts); + } + } + + private static void RegisterProductStatus( + string? rawEntry, + string status, + RedHatProductIndex productIndex, + IDictionary rpmPackages, + IDictionary baseProducts) + { + if (string.IsNullOrWhiteSpace(rawEntry) || !IsActionableStatus(status)) + { + return; + } + + var entry = rawEntry.Trim(); + var colonIndex = entry.IndexOf(':'); + if (colonIndex <= 0 || colonIndex == entry.Length - 1) + { + if (productIndex.TryGetValue(entry, out var baseOnly)) + { + var aggregate = baseProducts.TryGetValue(baseOnly.ProductId, out var existing) + ? existing + : new RedHatProductStatusEntry(baseOnly); + aggregate.Statuses.Add(status); + baseProducts[baseOnly.ProductId] = aggregate; + } + + return; + } + + var baseId = entry[..colonIndex]; + var packageId = entry[(colonIndex + 1)..]; + + if (productIndex.TryGetValue(baseId, out var baseNode)) + { + var aggregate = baseProducts.TryGetValue(baseNode.ProductId, out var existing) + ? existing + : new RedHatProductStatusEntry(baseNode); + aggregate.Statuses.Add(status); + baseProducts[baseNode.ProductId] = aggregate; + } + + if (!productIndex.TryGetValue(packageId, out var packageNode)) + { + return; + } + + var nevra = NormalizeNevra(packageNode.Name ?? packageNode.ProductId); + if (string.IsNullOrEmpty(nevra)) + { + return; + } + + var platform = baseProducts.TryGetValue(baseId, out var baseEntry) + ? baseEntry.Node.Name ?? baseId + : baseId; + + var key = string.Join('|', nevra, platform ?? string.Empty); + if (!rpmPackages.TryGetValue(key, out var rpm)) + { + rpm = new RedHatAffectedRpm(nevra, baseId, platform, packageNode.ProductId); + rpmPackages[key] = rpm; + } + + rpm.Statuses.Add(status); + } + + private static bool IsActionableStatus(string status) + { + return status.Equals(RedHatProductStatuses.Fixed, StringComparison.OrdinalIgnoreCase) + || status.Equals(RedHatProductStatuses.FirstFixed, StringComparison.OrdinalIgnoreCase) + || status.Equals(RedHatProductStatuses.KnownAffected, StringComparison.OrdinalIgnoreCase) + || status.Equals(RedHatProductStatuses.KnownNotAffected, StringComparison.OrdinalIgnoreCase) + || status.Equals(RedHatProductStatuses.UnderInvestigation, StringComparison.OrdinalIgnoreCase); + } + + private static IReadOnlyCollection BuildCvssMetrics( + string sourceName, + DateTimeOffset recordedAt, + string advisoryKey, + RedHatCsafEnvelope csaf) + { + var metrics = new List(); + if (csaf.Vulnerabilities is null) + { + return metrics; + } + + foreach (var vulnerability in csaf.Vulnerabilities) + { + if (vulnerability?.Scores is null) + { + continue; + } + + foreach (var score in vulnerability.Scores) + { + var cvss = score?.CvssV3; + if (cvss is null) + { + continue; + } + + if (!CvssMetricNormalizer.TryNormalize(cvss.Version, cvss.VectorString, cvss.BaseScore, cvss.BaseSeverity, out var normalized)) + { + continue; + } + + var provenance = new AdvisoryProvenance(sourceName, "cvss", vulnerability.Cve ?? advisoryKey, recordedAt); + metrics.Add(normalized.ToModel(provenance)); + } + } + + return metrics; + } + + private static string? NormalizeSeverity(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim().ToLowerInvariant() switch + { + "critical" => "critical", + "important" => "high", + "moderate" => "medium", + "low" => "low", + "none" => "none", + _ => value.Trim().ToLowerInvariant(), + }; + } + + private static string? TryGetMetadata(DocumentRecord document, string key) + { + if (document.Metadata is null) + { + return null; + } + + return document.Metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value) + ? value.Trim() + : null; + } + + private static string? NormalizeId(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private static string NormalizeNevra(string? value) + { + return string.IsNullOrWhiteSpace(value) + ? string.Empty + : value.Trim(); + } +} + +internal sealed class RedHatAffectedRpm +{ + public RedHatAffectedRpm(string nevra, string baseProductId, string? platform, string? productId) + { + Nevra = nevra; + BaseProductId = baseProductId; + Platform = platform; + ProductId = productId; + Statuses = new HashSet(StringComparer.OrdinalIgnoreCase); + } + + public string Nevra { get; } + + public string BaseProductId { get; } + + public string? Platform { get; } + + public string? ProductId { get; } + + public HashSet Statuses { get; } +} + +internal sealed class RedHatProductStatusEntry +{ + public RedHatProductStatusEntry(RedHatProductNode node) + { + Node = node; + Statuses = new HashSet(StringComparer.OrdinalIgnoreCase); + } + + public RedHatProductNode Node { get; } + + public HashSet Statuses { get; } +} + +internal static class RedHatProductStatuses +{ + public const string Fixed = "fixed"; + public const string FirstFixed = "first_fixed"; + public const string KnownAffected = "known_affected"; + public const string KnownNotAffected = "known_not_affected"; + public const string UnderInvestigation = "under_investigation"; +} + +internal sealed class RedHatProductIndex +{ + private readonly Dictionary _products; + + private RedHatProductIndex(Dictionary products) + { + _products = products; + } + + public static RedHatProductIndex Build(RedHatProductTree? tree) + { + var products = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (tree?.Branches is not null) + { + foreach (var branch in tree.Branches) + { + Traverse(branch, products); + } + } + + return new RedHatProductIndex(products); + } + + public bool TryGetValue(string productId, out RedHatProductNode node) + => _products.TryGetValue(productId, out node); + + private static void Traverse(RedHatProductBranch? branch, IDictionary products) + { + if (branch is null) + { + return; + } + + if (branch.Product is not null && !string.IsNullOrWhiteSpace(branch.Product.ProductId)) + { + var id = branch.Product.ProductId.Trim(); + products[id] = new RedHatProductNode( + id, + branch.Product.Name ?? branch.Name ?? id, + branch.Product.ProductIdentificationHelper?.Cpe, + branch.Product.ProductIdentificationHelper?.Purl); + } + + if (branch.Branches is null) + { + return; + } + + foreach (var child in branch.Branches) + { + Traverse(child, products); + } + } +} + +internal sealed record RedHatProductNode(string ProductId, string? Name, string? Cpe, string? Purl); diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs new file mode 100644 index 00000000..2f46e86c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs @@ -0,0 +1,66 @@ +using System; +using System.Text.Json; + +namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; + +internal readonly record struct RedHatSummaryItem(string AdvisoryId, DateTimeOffset ReleasedOn, Uri ResourceUri) +{ + private static readonly string[] AdvisoryFields = + { + "RHSA", + "RHBA", + "RHEA", + "RHUI", + "RHBG", + "RHBO", + "advisory" + }; + + public static bool TryParse(JsonElement element, out RedHatSummaryItem item) + { + item = default; + + string? advisoryId = null; + foreach (var field in AdvisoryFields) + { + if (element.TryGetProperty(field, out var advisoryProperty) && advisoryProperty.ValueKind == JsonValueKind.String) + { + var candidate = advisoryProperty.GetString(); + if (!string.IsNullOrWhiteSpace(candidate)) + { + advisoryId = candidate.Trim(); + break; + } + } + } + + if (string.IsNullOrWhiteSpace(advisoryId)) + { + return false; + } + + if (!element.TryGetProperty("released_on", out var releasedProperty) || releasedProperty.ValueKind != JsonValueKind.String) + { + return false; + } + + if (!DateTimeOffset.TryParse(releasedProperty.GetString(), out var releasedOn)) + { + return false; + } + + if (!element.TryGetProperty("resource_url", out var resourceProperty) || resourceProperty.ValueKind != JsonValueKind.String) + { + return false; + } + + var resourceValue = resourceProperty.GetString(); + if (!Uri.TryCreate(resourceValue, UriKind.Absolute, out var resourceUri)) + { + return false; + } + + item = new RedHatSummaryItem(advisoryId!, releasedOn.ToUniversalTime(), resourceUri); + return true; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs new file mode 100644 index 00000000..a01b038f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +internal static class RedHatJobKinds +{ + public const string Fetch = "source:redhat:fetch"; + public const string Parse = "source:redhat:parse"; + public const string Map = "source:redhat:map"; +} + +internal sealed class RedHatFetchJob : IJob +{ + private readonly RedHatConnector _connector; + + public RedHatFetchJob(RedHatConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class RedHatParseJob : IJob +{ + private readonly RedHatConnector _connector; + + public RedHatParseJob(RedHatConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class RedHatMapJob : IJob +{ + private readonly RedHatConnector _connector; + + public RedHatMapJob(RedHatConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs new file mode 100644 index 00000000..57a4a3af --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs @@ -0,0 +1,432 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Feedser.Source.Distro.RedHat.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +public sealed class RedHatConnector : IFeedConnector +{ + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly ILogger _logger; + private readonly RedHatOptions _options; + private readonly TimeProvider _timeProvider; + + public RedHatConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => RedHatConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var baseline = cursor.LastReleasedOn ?? now - _options.InitialBackfill; + var overlap = _options.Overlap > TimeSpan.Zero ? _options.Overlap : TimeSpan.Zero; + var afterThreshold = baseline - overlap; + if (afterThreshold < DateTimeOffset.UnixEpoch) + { + afterThreshold = DateTimeOffset.UnixEpoch; + } + + var processedSet = new HashSet(cursor.ProcessedAdvisoryIds, StringComparer.OrdinalIgnoreCase); + var newSummaries = new List(); + var stopDueToOlderData = false; + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var page = 1; page <= _options.MaxPagesPerFetch; page++) + { + var summaryUri = BuildSummaryUri(afterThreshold, page); + var summaryKey = summaryUri.ToString(); + touchedResources.Add(summaryKey); + + var cachedSummary = cursor.TryGetFetchCache(summaryKey); + var summaryMetadata = new Dictionary(StringComparer.Ordinal) + { + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["type"] = "summary" + }; + + var summaryRequest = new SourceFetchRequest(RedHatOptions.HttpClientName, SourceName, summaryUri) + { + Metadata = summaryMetadata, + ETag = cachedSummary?.ETag, + LastModified = cachedSummary?.LastModified, + TimeoutOverride = _options.FetchTimeout, + }; + + SourceFetchContentResult summaryResult; + try + { + summaryResult = await _fetchService.FetchContentAsync(summaryRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Red Hat Hydra summary fetch failed for {Uri}", summaryUri); + throw; + } + + if (summaryResult.IsNotModified) + { + if (page == 1) + { + break; + } + + continue; + } + + if (!summaryResult.IsSuccess || summaryResult.Content is null) + { + continue; + } + + cursor = cursor.WithFetchCache(summaryKey, summaryResult.ETag, summaryResult.LastModified); + + using var document = JsonDocument.Parse(summaryResult.Content); + + if (document.RootElement.ValueKind != JsonValueKind.Array) + { + _logger.LogWarning( + "Red Hat Hydra summary response had unexpected payload kind {Kind} for {Uri}", + document.RootElement.ValueKind, + summaryUri); + break; + } + + var pageCount = 0; + foreach (var element in document.RootElement.EnumerateArray()) + { + if (!RedHatSummaryItem.TryParse(element, out var summary)) + { + continue; + } + + pageCount++; + + if (cursor.LastReleasedOn.HasValue) + { + if (summary.ReleasedOn < cursor.LastReleasedOn.Value - overlap) + { + stopDueToOlderData = true; + break; + } + + if (summary.ReleasedOn < cursor.LastReleasedOn.Value) + { + stopDueToOlderData = true; + break; + } + + if (summary.ReleasedOn == cursor.LastReleasedOn.Value && processedSet.Contains(summary.AdvisoryId)) + { + continue; + } + } + + newSummaries.Add(summary); + processedSet.Add(summary.AdvisoryId); + + if (newSummaries.Count >= _options.MaxAdvisoriesPerFetch) + { + break; + } + } + + if (newSummaries.Count >= _options.MaxAdvisoriesPerFetch || stopDueToOlderData) + { + break; + } + + if (pageCount < _options.PageSize) + { + break; + } + } + + if (newSummaries.Count == 0) + { + return; + } + + newSummaries.Sort(static (left, right) => + { + var compare = left.ReleasedOn.CompareTo(right.ReleasedOn); + return compare != 0 + ? compare + : string.CompareOrdinal(left.AdvisoryId, right.AdvisoryId); + }); + + var pendingDocuments = new HashSet(cursor.PendingDocuments); + + foreach (var summary in newSummaries) + { + var resourceUri = summary.ResourceUri; + var resourceKey = resourceUri.ToString(); + touchedResources.Add(resourceKey); + + var cached = cursor.TryGetFetchCache(resourceKey); + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["advisoryId"] = summary.AdvisoryId, + ["releasedOn"] = summary.ReleasedOn.ToString("O", CultureInfo.InvariantCulture) + }; + + var request = new SourceFetchRequest(RedHatOptions.HttpClientName, SourceName, resourceUri) + { + Metadata = metadata, + ETag = cached?.ETag, + LastModified = cached?.LastModified, + TimeoutOverride = _options.FetchTimeout, + }; + + try + { + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified) + { + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + pendingDocuments.Add(result.Document.Id); + cursor = cursor.WithFetchCache(resourceKey, result.Document.Etag, result.Document.LastModified); + } + catch (Exception ex) + { + _logger.LogError(ex, "Red Hat Hydra advisory fetch failed for {Uri}", resourceUri); + throw; + } + } + + var maxRelease = newSummaries.Max(static item => item.ReleasedOn); + var idsForMaxRelease = newSummaries + .Where(item => item.ReleasedOn == maxRelease) + .Select(item => item.AdvisoryId) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + RedHatCursor updated; + if (cursor.LastReleasedOn.HasValue && maxRelease == cursor.LastReleasedOn.Value) + { + updated = cursor + .WithPendingDocuments(pendingDocuments) + .AddProcessedAdvisories(idsForMaxRelease) + .PruneFetchCache(touchedResources); + } + else + { + updated = cursor + .WithPendingDocuments(pendingDocuments) + .WithLastReleased(maxRelease, idsForMaxRelease) + .PruneFetchCache(touchedResources); + } + + await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingFetch = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + DocumentRecord? document = null; + + try + { + document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingFetch.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Red Hat document {DocumentId} missing GridFS content; skipping", document.Id); + remainingFetch.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + using var jsonDocument = JsonDocument.Parse(rawBytes); + var sanitized = JsonSerializer.Serialize(jsonDocument.RootElement); + var payload = BsonDocument.Parse(sanitized); + + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "redhat.csaf.v2", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingFetch.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + catch (Exception ex) + { + var uri = document?.Uri ?? documentId.ToString(); + _logger.LogError(ex, "Red Hat CSAF parse failed for {Uri}", uri); + remainingFetch.Remove(documentId); + pendingMappings.Remove(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingFetch) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + try + { + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var json = dto.Payload.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + }); + + using var jsonDocument = JsonDocument.Parse(json); + var advisory = RedHatMapper.Map(SourceName, dto, document, jsonDocument); + if (advisory is null) + { + pendingMappings.Remove(documentId); + continue; + } + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + catch (Exception ex) + { + _logger.LogError(ex, "Red Hat map failed for document {DocumentId}", documentId); + } + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return RedHatCursor.FromBsonDocument(record?.Cursor); + } + + private async Task UpdateCursorAsync(RedHatCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private Uri BuildSummaryUri(DateTimeOffset after, int page) + { + var builder = new UriBuilder(_options.BaseEndpoint); + var basePath = builder.Path?.TrimEnd('/') ?? string.Empty; + var summaryPath = _options.SummaryPath.TrimStart('/'); + builder.Path = string.IsNullOrEmpty(basePath) + ? $"/{summaryPath}" + : $"{basePath}/{summaryPath}"; + + var parameters = new Dictionary(StringComparer.Ordinal) + { + ["after"] = after.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), + ["per_page"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), + ["page"] = page.ToString(CultureInfo.InvariantCulture) + }; + + builder.Query = string.Join('&', parameters.Select(static kvp => + $"{Uri.EscapeDataString(kvp.Key)}={Uri.EscapeDataString(kvp.Value)}")); + return builder.Uri; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs new file mode 100644 index 00000000..62ac6115 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +public sealed class RedHatConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "redhat"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs new file mode 100644 index 00000000..d0c4e343 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +public sealed class RedHatDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:redhat"; + private const string FetchCron = "0,15,30,45 * * * *"; + private const string ParseCron = "5,20,35,50 * * * *"; + private const string MapCron = "10,25,40,55 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(12); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(15); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(20); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(6); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddRedHatConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var schedulerBuilder = new JobSchedulerBuilder(services); + + schedulerBuilder + .AddJob( + RedHatJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob( + RedHatJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob( + RedHatJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs new file mode 100644 index 00000000..92161bf7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs @@ -0,0 +1,34 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Distro.RedHat.Configuration; + +namespace StellaOps.Feedser.Source.Distro.RedHat; + +public static class RedHatServiceCollectionExtensions +{ + public static IServiceCollection AddRedHatConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(RedHatOptions.HttpClientName, (sp, httpOptions) => + { + var options = sp.GetRequiredService>().Value; + httpOptions.BaseAddress = options.BaseEndpoint; + httpOptions.Timeout = options.FetchTimeout; + httpOptions.UserAgent = options.UserAgent; + httpOptions.AllowedHosts.Clear(); + httpOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + httpOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj b/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj new file mode 100644 index 00000000..7af3a126 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj @@ -0,0 +1,15 @@ + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md b/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md new file mode 100644 index 00000000..81a35480 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md @@ -0,0 +1,15 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Hydra fetch with after= cursor|BE-Conn-RH|Source.Common|**DONE** – windowed paging with overlap, ETag/Last-Modified persisted.| +|DTOs for Security Data + OVAL|BE-Conn-RH|Tests|**DONE** – CSAF payloads serialized into `redhat.csaf.v2` DTOs.| +|NEVRA parser/comparer (complete)|BE-Conn-RH|Models|Covered by NevraTests; add edge cases.| +|Mapper to canonical rpm/cpe affected|BE-Conn-RH|Models|**DONE** – maps fixed/known ranges, CPE provenance, status ranges.| +|Job scheduler registration aligns with Options pipeline|BE-Conn-RH|Core|**DONE** – registered fetch/parse/map via JobSchedulerBuilder, preserving option overrides and tightening cron/timeouts.| +|Watermark persistence + resume|BE-Conn-RH|Storage.Mongo|**DONE** – cursor updates via SourceStateRepository.| +|Precedence tests vs NVD|QA|Merge|**DONE** – Added AffectedPackagePrecedenceResolver + tests ensuring Red Hat CPEs override NVD ranges.| +|Golden mapping fixtures|QA|Fixtures|DOING – added RHSA-2025:0002/0003 fixtures; need validation pass once connector regression fixed.| +|Job scheduling defaults for source:redhat tasks|BE-Core|JobScheduler|**DONE** – Cron windows + per-job timeouts defined for fetch/parse/map.| +|Express unaffected/investigation statuses without overloading range fields|BE-Conn-RH|Models|**DONE** – Introduced AffectedPackageStatus collection and updated mapper/tests.| +|Reference dedupe & ordering in mapper|BE-Conn-RH|Models|DONE – mapper consolidates by URL, merges metadata, deterministic ordering validated in tests.| +|Hydra summary fetch through SourceFetchService|BE-Conn-RH|Source.Common|DONE – summary pages now fetched via SourceFetchService with cache + conditional headers.| diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Class1.cs b/src/StellaOps.Feedser.Source.Distro.Suse/Class1.cs new file mode 100644 index 00000000..4bf1a97f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Suse; + +public sealed class DistroSuseConnectorPlugin : IConnectorPlugin +{ + public string Name => "distro-suse"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj b/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Class1.cs b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Class1.cs new file mode 100644 index 00000000..26e52c3d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Distro.Ubuntu; + +public sealed class DistroUbuntuConnectorPlugin : IConnectorPlugin +{ + public string Name => "distro-ubuntu"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj b/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ghsa/Class1.cs b/src/StellaOps.Feedser.Source.Ghsa/Class1.cs new file mode 100644 index 00000000..93461ef6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ghsa; + +public sealed class GhsaConnectorPlugin : IConnectorPlugin +{ + public string Name => "ghsa"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj b/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Class1.cs b/src/StellaOps.Feedser.Source.Ics.Cisa/Class1.cs new file mode 100644 index 00000000..41ce2d5e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Cisa/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ics.Cisa; + +public sealed class IcsCisaConnectorPlugin : IConnectorPlugin +{ + public string Name => "ics-cisa"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj b/src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html new file mode 100644 index 00000000..166b6423 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html @@ -0,0 +1,18 @@ + + + + + ACME Corp controllers multiple vulnerabilities + + +
    +

    ACME Corp controllers multiple vulnerabilities

    +

    Researchers identified that ACME Corp ICS controller models X100 and X200 are affected by improper access controls.

    +

    Exploitation of CVE-2024-7777 can allow authenticated attackers to execute arbitrary commands. Additional details are provided in CVE-2024-8888.

    +
      +
    • Vendor: ACME Corp
      • +
    • Affected models: X100, X200
      • +
    +
    + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json new file mode 100644 index 00000000..40a7a5cf --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json @@ -0,0 +1,235 @@ +{ + "advisoryKey": "acme-controller-2024", + "affectedPackages": [ + { + "identifier": "2024", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "2024" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "7777 can allow authenticated attackers to execute arbitrary commands", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "7777 can allow authenticated attackers to execute arbitrary commands" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "7777)", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "7777)" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "8888", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "8888" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "ACME Corp", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "ACME Corp Affected models", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp Affected models" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "ACME Corp industrial", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "ACME Corp industrial" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "Additional details are provided in CVE", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Additional details are provided in CVE" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "Exploitation of CVE", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Exploitation of CVE" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "Vendor", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "Vendor" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + }, + { + "identifier": "X100, X200", + "platform": null, + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "X100, X200" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [] + } + ], + "aliases": [ + "CVE-2024-7777", + "CVE-2024-8888", + "acme-controller-2024" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-10-15T10:00:00+00:00", + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-10-20T00:00:00+00:00", + "source": "ics-kaspersky", + "value": "https://ics-cert.example/advisories/acme-controller-2024/" + }, + { + "kind": "mapping", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "acme-controller-2024" + } + ], + "published": "2024-10-15T10:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "https://ics-cert.example/advisories/acme-controller-2024/" + }, + "sourceTag": "kaspersky-ics", + "summary": null, + "url": "https://ics-cert.example/advisories/acme-controller-2024/" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-7777" + }, + "sourceTag": "CVE-2024-7777", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-7777" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-10-20T00:01:00+00:00", + "source": "ics-kaspersky", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-8888" + }, + "sourceTag": "CVE-2024-8888", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-8888" + } + ], + "severity": null, + "summary": "ACME Corp industrial controllers allow remote compromise (CVE-2024-7777).", + "title": "ACME Corp controllers multiple vulnerabilities" +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml new file mode 100644 index 00000000..1f7fd270 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml @@ -0,0 +1,17 @@ + + + + Kaspersky ICS CERT - Advisories + https://ics-cert.kaspersky.com/feed-advisories/ + Test feed + + ACME Corp controllers multiple vulnerabilities + https://ics-cert.example/advisories/acme-controller-2024/ + + Tue, 15 Oct 2024 10:00:00 +0000 + Kaspersky ICS CERT + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs new file mode 100644 index 00000000..d0dcb9f2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs @@ -0,0 +1,338 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Ics.Kaspersky; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Tests; + +[Collection("mongo-fixture")] +public sealed class KasperskyConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public KasperskyConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 10, 20, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_CreatesSnapshot() + { + var options = new KasperskyOptions + { + FeedUri = new Uri("https://ics-cert.example/feed-advisories/", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(1), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + _handler.Clear(); + + _handler.AddTextResponse(options.FeedUri, ReadFixture("feed-page1.xml"), "application/rss+xml"); + var detailUri = new Uri("https://ics-cert.example/advisories/acme-controller-2024/"); + _handler.AddTextResponse(detailUri, ReadFixture("detail-acme-controller-2024.html"), "text/html"); + + var connector = new KasperskyConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + Assert.Single(advisories); + var canonical = SnapshotSerializer.ToSnapshot(advisories.Single()); + var expected = ReadFixture("expected-advisory.json"); + var normalizedExpected = NormalizeLineEndings(expected); + var normalizedActual = NormalizeLineEndings(canonical); + if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Ics", "Kaspersky", "Fixtures", "expected-advisory.actual.json"); + File.WriteAllText(actualPath, canonical); + } + + Assert.Equal(normalizedExpected, normalizedActual); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KasperskyConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pending) + ? pending.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocuments); + } + + [Fact] + public async Task FetchFailure_RecordsBackoff() + { + var options = new KasperskyOptions + { + FeedUri = new Uri("https://ics-cert.example/feed-advisories/", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(1), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + _handler.AddResponse(options.FeedUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("feed error", Encoding.UTF8, "text/plain"), + }); + + var connector = new KasperskyConnectorPlugin().Create(provider); + + await Assert.ThrowsAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KasperskyConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal(1, state!.FailCount); + Assert.NotNull(state.LastFailureReason); + Assert.Contains("500", state.LastFailureReason, StringComparison.Ordinal); + Assert.True(state.BackoffUntil.HasValue); + Assert.True(state.BackoffUntil!.Value > _timeProvider.GetUtcNow()); + } + + [Fact] + public async Task Fetch_NotModifiedMaintainsDocumentState() + { + var options = new KasperskyOptions + { + FeedUri = new Uri("https://ics-cert.example/feed-advisories/", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(1), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + + var feedXml = ReadFixture("feed-page1.xml"); + var detailUri = new Uri("https://ics-cert.example/advisories/acme-controller-2024/"); + var detailHtml = ReadFixture("detail-acme-controller-2024.html"); + var etag = new EntityTagHeaderValue("\"ics-2024-acme\""); + var lastModified = new DateTimeOffset(2024, 10, 15, 10, 0, 0, TimeSpan.Zero); + + _handler.AddTextResponse(options.FeedUri, feedXml, "application/rss+xml"); + _handler.AddResponse(detailUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(detailHtml, Encoding.UTF8, "text/html"), + }; + response.Headers.ETag = etag; + response.Content.Headers.LastModified = lastModified; + return response; + }); + + var connector = new KasperskyConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + _handler.AddTextResponse(options.FeedUri, feedXml, "application/rss+xml"); + _handler.AddResponse(detailUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = etag; + return response; + }); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KasperskyConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Equal(0, pendingDocs.AsBsonArray.Count); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)); + Assert.Equal(0, pendingMappings.AsBsonArray.Count); + } + + [Fact] + public async Task Fetch_DuplicateContentSkipsRequeue() + { + var options = new KasperskyOptions + { + FeedUri = new Uri("https://ics-cert.example/feed-advisories/", UriKind.Absolute), + WindowSize = TimeSpan.FromDays(30), + WindowOverlap = TimeSpan.FromDays(1), + MaxPagesPerFetch = 1, + RequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + _handler.Clear(); + + var feedXml = ReadFixture("feed-page1.xml"); + var detailUri = new Uri("https://ics-cert.example/advisories/acme-controller-2024/"); + var detailHtml = ReadFixture("detail-acme-controller-2024.html"); + + _handler.AddTextResponse(options.FeedUri, feedXml, "application/rss+xml"); + _handler.AddTextResponse(detailUri, detailHtml, "text/html"); + + var connector = new KasperskyConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + _handler.AddTextResponse(options.FeedUri, feedXml, "application/rss+xml"); + _handler.AddTextResponse(detailUri, detailHtml, "text/html"); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + document = await documentStore.FindBySourceAndUriAsync(KasperskyConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KasperskyConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocs = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocs); + var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingMappings); + } + + private async Task EnsureServiceProviderAsync(KasperskyOptions template) + { + if (_serviceProvider is not null) + { + await ResetDatabaseAsync(); + return; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddKasperskyIcsConnector(opts => + { + opts.FeedUri = template.FeedUri; + opts.WindowSize = template.WindowSize; + opts.WindowOverlap = template.WindowOverlap; + opts.MaxPagesPerFetch = template.MaxPagesPerFetch; + opts.RequestDelay = template.RequestDelay; + }); + + services.Configure(KasperskyOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + _serviceProvider = services.BuildServiceProvider(); + var bootstrapper = _serviceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + } + + private Task ResetDatabaseAsync() + => _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Ics", "Kaspersky", "Fixtures", filename); + return File.ReadAllText(path); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider?.Dispose(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj new file mode 100644 index 00000000..57f706c3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md b/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md new file mode 100644 index 00000000..be150c72 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Kaspersky ICS-CERT connector; authoritative for OT/ICS vendor advisories covered by Kaspersky ICS-CERT; maps affected products as ICS domain entities with platform tags. +## Scope +- Discover/fetch advisories list; window by publish date or slug; fetch detail pages; handle pagination. +- Validate HTML or JSON; extract CVEs, affected OT vendors/models/families, mitigations; normalize product taxonomy; map fixed versions if present. +- Persist raw docs with sha256; maintain source_state; idempotent mapping. +## Participants +- Source.Common (HTTP, HTML helpers, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, source_state). +- Models (canonical; affected.platform="ics-vendor", tags for device families). +- Core/WebService (jobs: source:ics-kaspersky:fetch|parse|map). +- Merge engine respects ICS vendor authority for OT impact. +## Interfaces & contracts +- Aliases: CVE ids; if stable ICS-CERT advisory id exists, store scheme "ICS-KASP". +- Affected: Type=vendor; Vendor/Product populated; platforms/tags for device family or firmware line; versions with fixedBy when explicit. +- References: advisory, vendor pages, mitigation guides; typed; deduped. +- Provenance: method=parser; value=advisory slug. +## In/Out of scope +In: ICS advisory mapping, affected vendor products, mitigation references. +Out: firmware downloads; reverse-engineering artifacts. +## Observability & security expectations +- Metrics: icsk.fetch.items, icsk.parse.fail, icsk.map.affected_count. +- Logs: slugs, vendor/product counts, timing; allowlist host. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Ics.Kaspersky.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs new file mode 100644 index 00000000..19f36e03 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs @@ -0,0 +1,53 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; + +public sealed class KasperskyOptions +{ + public static string HttpClientName => "source.ics.kaspersky"; + + public Uri FeedUri { get; set; } = new("https://ics-cert.kaspersky.com/feed-advisories/", UriKind.Absolute); + + public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2); + + public int MaxPagesPerFetch { get; set; } = 3; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500); + + [MemberNotNull(nameof(FeedUri))] + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new InvalidOperationException("FeedUri must be an absolute URI."); + } + + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("WindowSize must be greater than zero."); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new InvalidOperationException("WindowOverlap cannot be negative."); + } + + if (WindowOverlap >= WindowSize) + { + throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxPagesPerFetch must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs new file mode 100644 index 00000000..dc61ee38 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +internal sealed record KasperskyAdvisoryDto( + string AdvisoryKey, + string Title, + string Link, + DateTimeOffset Published, + string? Summary, + string Content, + ImmutableArray CveIds, + ImmutableArray VendorNames); diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs new file mode 100644 index 00000000..006e825c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +internal static class KasperskyAdvisoryParser +{ + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d+", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled); + + public static KasperskyAdvisoryDto Parse( + string advisoryKey, + string title, + string link, + DateTimeOffset published, + string? summary, + byte[] rawHtml) + { + var content = ExtractText(rawHtml); + var cves = ExtractCves(title, summary, content); + var vendors = ExtractVendors(title, summary, content); + + return new KasperskyAdvisoryDto( + advisoryKey, + title, + link, + published, + summary, + content, + cves, + vendors); + } + + private static string ExtractText(byte[] rawHtml) + { + if (rawHtml.Length == 0) + { + return string.Empty; + } + + var html = Encoding.UTF8.GetString(rawHtml); + html = Regex.Replace(html, "", string.Empty, RegexOptions.IgnoreCase); + html = Regex.Replace(html, "", string.Empty, RegexOptions.IgnoreCase); + html = Regex.Replace(html, "", string.Empty, RegexOptions.Singleline); + html = Regex.Replace(html, "<[^>]+>", " "); + var decoded = System.Net.WebUtility.HtmlDecode(html); + return string.IsNullOrWhiteSpace(decoded) ? string.Empty : WhitespaceRegex.Replace(decoded, " ").Trim(); + } + + private static ImmutableArray ExtractCves(string title, string? summary, string content) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + void Capture(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + foreach (Match match in CveRegex.Matches(text)) + { + if (match.Success) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + } + + Capture(title); + Capture(summary); + Capture(content); + + return set.OrderBy(static cve => cve, StringComparer.Ordinal).ToImmutableArray(); + } + + private static ImmutableArray ExtractVendors(string title, string? summary, string content) + { + var candidates = new HashSet(StringComparer.OrdinalIgnoreCase); + + void AddCandidate(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + foreach (var segment in SplitSegments(text)) + { + var cleaned = CleanVendorSegment(segment); + if (!string.IsNullOrWhiteSpace(cleaned)) + { + candidates.Add(cleaned); + } + } + } + + AddCandidate(title); + AddCandidate(summary); + AddCandidate(content); + + return candidates.Count == 0 + ? ImmutableArray.Empty + : candidates + .OrderBy(static vendor => vendor, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static IEnumerable SplitSegments(string text) + { + var separators = new[] { ".", "-", "–", "—", ":" }; + var queue = new Queue(); + queue.Enqueue(text); + + foreach (var separator in separators) + { + var count = queue.Count; + for (var i = 0; i < count; i++) + { + var item = queue.Dequeue(); + var parts = item.Split(separator, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + foreach (var part in parts) + { + queue.Enqueue(part); + } + } + } + + return queue; + } + + private static string? CleanVendorSegment(string value) + { + var trimmed = value.Trim(); + if (string.IsNullOrEmpty(trimmed)) + { + return null; + } + + var lowered = trimmed.ToLowerInvariant(); + if (lowered.Contains("cve-", StringComparison.Ordinal) || lowered.Contains("vulnerability", StringComparison.Ordinal)) + { + trimmed = trimmed.Split(new[] { "vulnerability", "vulnerabilities" }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault() ?? trimmed; + } + + var providedMatch = Regex.Match(trimmed, "provided by\\s+(?[A-Za-z0-9&.,' ]+)", RegexOptions.IgnoreCase); + if (providedMatch.Success) + { + trimmed = providedMatch.Groups["vendor"].Value; + } + + var descriptorMatch = Regex.Match(trimmed, "^(?[A-Z][A-Za-z0-9&.,' ]{1,80}?)(?:\\s+(?:controllers?|devices?|modules?|products?|gateways?|routers?|appliances?|systems?|solutions?|firmware))\\b", RegexOptions.IgnoreCase); + if (descriptorMatch.Success) + { + trimmed = descriptorMatch.Groups["vendor"].Value; + } + + trimmed = trimmed.Replace("’", "'", StringComparison.Ordinal); + trimmed = trimmed.Replace("\"", string.Empty, StringComparison.Ordinal); + + if (trimmed.Length > 200) + { + trimmed = trimmed[..200]; + } + + return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs new file mode 100644 index 00000000..b5879e8e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs @@ -0,0 +1,207 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +internal sealed record KasperskyCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyFetchCache = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static KasperskyCursor Empty { get; } = new(null, EmptyGuidList, EmptyGuidList, EmptyFetchCache); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + if (FetchCache.Count > 0) + { + var cacheArray = new BsonArray(); + foreach (var (uri, metadata) in FetchCache) + { + var cacheDocument = new BsonDocument + { + ["uri"] = uri, + }; + + if (!string.IsNullOrWhiteSpace(metadata.ETag)) + { + cacheDocument["etag"] = metadata.ETag; + } + + if (metadata.LastModified.HasValue) + { + cacheDocument["lastModified"] = metadata.LastModified.Value.UtcDateTime; + } + + cacheArray.Add(cacheDocument); + } + + document["fetchCache"] = cacheArray; + } + + return document; + } + + public static KasperskyCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPublished = document.TryGetValue("lastPublished", out var lastPublishedValue) + ? ParseDate(lastPublishedValue) + : null; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new KasperskyCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache); + } + + public KasperskyCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public KasperskyCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public KasperskyCursor WithLastPublished(DateTimeOffset? timestamp) + => this with { LastPublished = timestamp }; + + public KasperskyCursor WithFetchMetadata(string requestUri, string? etag, DateTimeOffset? lastModified) + { + if (string.IsNullOrWhiteSpace(requestUri)) + { + return this; + } + + var cache = new Dictionary(FetchCache, StringComparer.OrdinalIgnoreCase) + { + [requestUri] = new KasperskyFetchMetadata(etag, lastModified), + }; + + return this with { FetchCache = cache }; + } + + public KasperskyCursor PruneFetchCache(IEnumerable keepUris) + { + if (FetchCache.Count == 0) + { + return this; + } + + var keepSet = new HashSet(keepUris ?? Array.Empty(), StringComparer.OrdinalIgnoreCase); + if (keepSet.Count == 0) + { + return this; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var uri in keepSet) + { + if (FetchCache.TryGetValue(uri, out var metadata)) + { + cache[uri] = metadata; + } + } + + return this with { FetchCache = cache }; + } + + public bool TryGetFetchMetadata(string requestUri, out KasperskyFetchMetadata metadata) + { + if (FetchCache.TryGetValue(requestUri, out metadata!)) + { + return true; + } + + metadata = default!; + return false; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonArray array) + { + return EmptyFetchCache; + } + + var cache = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in array) + { + if (element is not BsonDocument cacheDocument) + { + continue; + } + + if (!cacheDocument.TryGetValue("uri", out var uriValue) || uriValue.BsonType != BsonType.String) + { + continue; + } + + var uri = uriValue.AsString; + string? etag = cacheDocument.TryGetValue("etag", out var etagValue) && etagValue.IsString ? etagValue.AsString : null; + DateTimeOffset? lastModified = cacheDocument.TryGetValue("lastModified", out var lastModifiedValue) + ? ParseDate(lastModifiedValue) + : null; + + cache[uri] = new KasperskyFetchMetadata(etag, lastModified); + } + + return cache.Count == 0 ? EmptyFetchCache : cache; + } +} + +internal sealed record KasperskyFetchMetadata(string? ETag, DateTimeOffset? LastModified); diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs new file mode 100644 index 00000000..6c565149 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +public sealed class KasperskyFeedClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly KasperskyOptions _options; + private readonly ILogger _logger; + + private static readonly XNamespace ContentNamespace = "http://purl.org/rss/1.0/modules/content/"; + + public KasperskyFeedClient(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> GetItemsAsync(int page, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(KasperskyOptions.HttpClientName); + var feedUri = BuildUri(_options.FeedUri, page); + + using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var reader = new StreamReader(stream, Encoding.UTF8); + var xml = await reader.ReadToEndAsync().ConfigureAwait(false); + + var document = XDocument.Parse(xml, LoadOptions.None); + var items = new List(); + var channel = document.Root?.Element("channel"); + if (channel is null) + { + _logger.LogWarning("Feed {FeedUri} is missing channel element", feedUri); + return items; + } + + foreach (var item in channel.Elements("item")) + { + var title = item.Element("title")?.Value?.Trim(); + var linkValue = item.Element("link")?.Value?.Trim(); + var pubDateValue = item.Element("pubDate")?.Value?.Trim(); + var summary = item.Element("description")?.Value?.Trim(); + + if (string.IsNullOrWhiteSpace(title) || string.IsNullOrWhiteSpace(linkValue) || string.IsNullOrWhiteSpace(pubDateValue)) + { + continue; + } + + if (!Uri.TryCreate(linkValue, UriKind.Absolute, out var link)) + { + _logger.LogWarning("Skipping feed item with invalid link: {Link}", linkValue); + continue; + } + + if (!DateTimeOffset.TryParse(pubDateValue, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var published)) + { + _logger.LogWarning("Skipping feed item {Title} due to invalid pubDate {PubDate}", title, pubDateValue); + continue; + } + + var encoded = item.Element(ContentNamespace + "encoded")?.Value; + if (!string.IsNullOrWhiteSpace(encoded)) + { + summary ??= HtmlToPlainText(encoded); + } + + items.Add(new KasperskyFeedItem(title, Canonicalize(link), published.ToUniversalTime(), summary)); + } + + return items; + } + + private static Uri BuildUri(Uri baseUri, int page) + { + if (page <= 1) + { + return baseUri; + } + + var builder = new UriBuilder(baseUri); + var trimmed = builder.Query.TrimStart('?'); + var pageSegment = $"paged={page.ToString(CultureInfo.InvariantCulture)}"; + builder.Query = string.IsNullOrEmpty(trimmed) + ? pageSegment + : $"{trimmed}&{pageSegment}"; + return builder.Uri; + } + + private static Uri Canonicalize(Uri link) + { + if (string.IsNullOrEmpty(link.Query)) + { + return link; + } + + var builder = new UriBuilder(link) + { + Query = string.Empty, + }; + return builder.Uri; + } + + private static string? HtmlToPlainText(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return null; + } + + var withoutScripts = System.Text.RegularExpressions.Regex.Replace(html, "", string.Empty, System.Text.RegularExpressions.RegexOptions.IgnoreCase); + var withoutStyles = System.Text.RegularExpressions.Regex.Replace(withoutScripts, "", string.Empty, System.Text.RegularExpressions.RegexOptions.IgnoreCase); + var withoutTags = System.Text.RegularExpressions.Regex.Replace(withoutStyles, "<[^>]+>", " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags); + return string.IsNullOrWhiteSpace(decoded) ? null : System.Text.RegularExpressions.Regex.Replace(decoded, "\\s+", " ").Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs new file mode 100644 index 00000000..8e08b1bb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs @@ -0,0 +1,9 @@ +using System; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +public sealed record KasperskyFeedItem( + string Title, + Uri Link, + DateTimeOffset Published, + string? Summary); diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs new file mode 100644 index 00000000..0d05a3d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +internal static class KasperskyJobKinds +{ + public const string Fetch = "source:ics-kaspersky:fetch"; + public const string Parse = "source:ics-kaspersky:parse"; + public const string Map = "source:ics-kaspersky:map"; +} + +internal sealed class KasperskyFetchJob : IJob +{ + private readonly KasperskyConnector _connector; + + public KasperskyFetchJob(KasperskyConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class KasperskyParseJob : IJob +{ + private readonly KasperskyConnector _connector; + + public KasperskyParseJob(KasperskyConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class KasperskyMapJob : IJob +{ + private readonly KasperskyConnector _connector; + + public KasperskyMapJob(KasperskyConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs new file mode 100644 index 00000000..acc36227 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs @@ -0,0 +1,445 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +using StellaOps.Feedser.Source.Ics.Kaspersky.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +public sealed class KasperskyConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General) + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly KasperskyFeedClient _feedClient; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly KasperskyOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public KasperskyConnector( + KasperskyFeedClient feedClient, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => KasperskyConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var windowStart = cursor.LastPublished.HasValue + ? cursor.LastPublished.Value - _options.WindowOverlap + : now - _options.WindowSize; + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var maxPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + var cursorState = cursor; + var touchedResources = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var page = 1; page <= _options.MaxPagesPerFetch; page++) + { + IReadOnlyList items; + try + { + items = await _feedClient.GetItemsAsync(page, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Kaspersky ICS feed page {Page}", page); + await _stateRepository.MarkFailureAsync( + SourceName, + now, + TimeSpan.FromMinutes(5), + ex.Message, + cancellationToken).ConfigureAwait(false); + throw; + } + if (items.Count == 0) + { + break; + } + + foreach (var item in items) + { + if (item.Published < windowStart) + { + page = _options.MaxPagesPerFetch + 1; + break; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["kaspersky.title"] = item.Title, + ["kaspersky.link"] = item.Link.ToString(), + ["kaspersky.published"] = item.Published.ToString("O"), + }; + + if (!string.IsNullOrWhiteSpace(item.Summary)) + { + metadata["kaspersky.summary"] = item.Summary!; + } + + var slug = ExtractSlug(item.Link); + if (!string.IsNullOrWhiteSpace(slug)) + { + metadata["kaspersky.slug"] = slug; + } + + var resourceKey = item.Link.ToString(); + touchedResources.Add(resourceKey); + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, resourceKey, cancellationToken).ConfigureAwait(false); + + var fetchRequest = new SourceFetchRequest(KasperskyOptions.HttpClientName, SourceName, item.Link) + { + Metadata = metadata, + }; + + if (cursorState.TryGetFetchMetadata(resourceKey, out var cachedFetch)) + { + fetchRequest = fetchRequest with + { + ETag = cachedFetch.ETag, + LastModified = cachedFetch.LastModified, + }; + } + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to fetch Kaspersky advisory {Link}", item.Link); + await _stateRepository.MarkFailureAsync( + SourceName, + _timeProvider.GetUtcNow(), + TimeSpan.FromMinutes(5), + ex.Message, + cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.IsNotModified) + { + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (existing is not null + && string.Equals(existing.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase) + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + await _documentStore.UpdateStatusAsync(result.Document.Id, existing.Status, cancellationToken).ConfigureAwait(false); + cursorState = cursorState.WithFetchMetadata(resourceKey, result.Document.Etag, result.Document.LastModified); + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + + continue; + } + + pendingDocuments.Add(result.Document.Id); + cursorState = cursorState.WithFetchMetadata(resourceKey, result.Document.Etag, result.Document.LastModified); + if (item.Published > maxPublished) + { + maxPublished = item.Published; + } + } + } + + cursorState = cursorState.PruneFetchCache(touchedResources); + + var updatedCursor = cursorState + .WithPendingDocuments(pendingDocuments) + .WithLastPublished(maxPublished == DateTimeOffset.MinValue ? cursor.LastPublished : maxPublished); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Kaspersky document {DocumentId} missing GridFS content", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var metadata = document.Metadata ?? new Dictionary(); + var title = metadata.TryGetValue("kaspersky.title", out var titleValue) ? titleValue : document.Uri; + var link = metadata.TryGetValue("kaspersky.link", out var linkValue) ? linkValue : document.Uri; + var published = metadata.TryGetValue("kaspersky.published", out var publishedValue) && DateTimeOffset.TryParse(publishedValue, out var parsedPublished) + ? parsedPublished.ToUniversalTime() + : document.FetchedAt; + var summary = metadata.TryGetValue("kaspersky.summary", out var summaryValue) ? summaryValue : null; + var slug = metadata.TryGetValue("kaspersky.slug", out var slugValue) ? slugValue : ExtractSlug(new Uri(link, UriKind.Absolute)); + var advisoryKey = string.IsNullOrWhiteSpace(slug) ? Guid.NewGuid().ToString("N") : slug; + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed downloading raw Kaspersky document {DocumentId}", document.Id); + throw; + } + + var dto = KasperskyAdvisoryParser.Parse(advisoryKey, title, link, published, summary, rawBytes); + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ics.kaspersky/1", payload, _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + _logger.LogWarning("Skipping Kaspersky mapping for {DocumentId}: DTO or document missing", documentId); + pendingMappings.Remove(documentId); + continue; + } + + var dtoJson = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + KasperskyAdvisoryDto advisoryDto; + try + { + advisoryDto = JsonSerializer.Deserialize(dtoJson, SerializerOptions) + ?? throw new InvalidOperationException("Deserialized DTO was null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize Kaspersky DTO for {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var fetchProvenance = new AdvisoryProvenance(SourceName, "document", document.Uri, document.FetchedAt); + var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", advisoryDto.AdvisoryKey, dto.ValidatedAt); + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + advisoryDto.AdvisoryKey, + }; + foreach (var cve in advisoryDto.CveIds) + { + aliases.Add(cve); + } + + var references = new List(); + try + { + references.Add(new AdvisoryReference( + advisoryDto.Link, + "advisory", + "kaspersky-ics", + null, + new AdvisoryProvenance(SourceName, "reference", advisoryDto.Link, dto.ValidatedAt))); + } + catch (ArgumentException) + { + _logger.LogWarning("Invalid advisory link {Link} for {AdvisoryKey}", advisoryDto.Link, advisoryDto.AdvisoryKey); + } + + foreach (var cve in advisoryDto.CveIds) + { + var url = $"https://www.cve.org/CVERecord?id={cve}"; + try + { + references.Add(new AdvisoryReference( + url, + "advisory", + cve, + null, + new AdvisoryProvenance(SourceName, "reference", url, dto.ValidatedAt))); + } + catch (ArgumentException) + { + // ignore malformed + } + } + + var affectedPackages = new List(); + foreach (var vendor in advisoryDto.VendorNames) + { + var provenance = new[] + { + new AdvisoryProvenance(SourceName, "affected", vendor, dto.ValidatedAt) + }; + affectedPackages.Add(new AffectedPackage( + AffectedPackageTypes.IcsVendor, + vendor, + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: provenance)); + } + + var advisory = new Advisory( + advisoryDto.AdvisoryKey, + advisoryDto.Title, + advisoryDto.Summary ?? advisoryDto.Content, + language: "en", + published: advisoryDto.Published, + modified: advisoryDto.Published, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mappingProvenance }); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? KasperskyCursor.Empty : KasperskyCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(KasperskyCursor cursor, CancellationToken cancellationToken) + { + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static string? ExtractSlug(Uri link) + { + var segments = link.Segments; + if (segments.Length == 0) + { + return null; + } + + var last = segments[^1].Trim('/'); + return string.IsNullOrWhiteSpace(last) && segments.Length > 1 ? segments[^2].Trim('/') : last; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs new file mode 100644 index 00000000..400ea3de --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +public sealed class KasperskyConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "ics-kaspersky"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs new file mode 100644 index 00000000..d2bb12ad --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +public sealed class KasperskyDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:ics-kaspersky"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddKasperskyIcsConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, KasperskyJobKinds.Fetch, typeof(KasperskyFetchJob)); + EnsureJob(options, KasperskyJobKinds.Parse, typeof(KasperskyParseJob)); + EnsureJob(options, KasperskyJobKinds.Map, typeof(KasperskyMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs new file mode 100644 index 00000000..28075872 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +using StellaOps.Feedser.Source.Ics.Kaspersky.Internal; + +namespace StellaOps.Feedser.Source.Ics.Kaspersky; + +public static class KasperskyServiceCollectionExtensions +{ + public static IServiceCollection AddKasperskyIcsConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(KasperskyOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.FeedUri; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.IcsKaspersky/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.FeedUri.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/rss+xml"; + }); + + services.AddTransient(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj b/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj new file mode 100644 index 00000000..07f798f6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md b/src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md new file mode 100644 index 00000000..b98e45c2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md @@ -0,0 +1,10 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|List/detail fetcher with windowing|BE-Conn-ICS-Kaspersky|Source.Common|**DONE** – feed client paginates and fetches detail pages with window overlap.| +|Extractor (vendors/models/CVEs)|BE-Conn-ICS-Kaspersky|Source.Common|**DONE** – parser normalizes vendor/model taxonomy into DTO.| +|DTO validation and sanitizer|BE-Conn-ICS-Kaspersky, QA|Source.Common|**DONE** – HTML parsed into DTO with sanitizer guardrails.| +|Canonical mapping (affected, refs)|BE-Conn-ICS-Kaspersky|Models|**DONE** – mapper outputs `ics-vendor` affected entries with provenance.| +|State/dedupe and fixtures|BE-Conn-ICS-Kaspersky, QA|Storage.Mongo|**DONE** – duplicate-content and resume tests exercise SHA gating + cursor hygiene.| +|Backoff on fetch failures|BE-Conn-ICS-Kaspersky|Storage.Mongo|**DONE** – feed/page failures mark source_state with timed backoff.| +|Conditional fetch caching|BE-Conn-ICS-Kaspersky|Source.Common|**DONE** – fetch cache persists ETag/Last-Modified; not-modified scenarios validated in tests.| diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json new file mode 100644 index 00000000..3de2ff97 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json @@ -0,0 +1,84 @@ +{ + "affectedPackages": [ + { + "identifier": "cpe:2.3:o:example:imaginary_controller_firmware:2.0", + "platform": "Example Industrial Corporation", + "provenance": [ + { + "kind": "affected", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "cpe:2.3:o:example:imaginary_controller_firmware:2.0" + } + ], + "type": "cpe", + "versionRanges": [] + } + ], + "aliases": [ + "CVE-2024-5555", + "JVNDB-2024-123456" + ], + "advisoryKey": "JVNDB-2024-123456", + "cvssMetrics": [ + { + "baseScore": 8.8, + "baseSeverity": "high", + "provenance": { + "kind": "cvss", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "Base" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-03-10T02:30:00+00:00", + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-03-10T00:00:00+00:00", + "source": "jvn", + "value": "https://jvndb.jvn.jp/en/contents/2024/JVNDB-2024-123456.html" + }, + { + "kind": "mapping", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "JVNDB-2024-123456" + } + ], + "published": "2024-03-09T02:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://vendor.example.com/advisories/EX-2024-01" + }, + "sourceTag": "EX-2024-01", + "summary": "Example ICS Vendor Advisory", + "url": "https://vendor.example.com/advisories/EX-2024-01" + }, + { + "kind": "advisory", + "provenance": { + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-5555" + }, + "sourceTag": "CVE-2024-5555", + "summary": "Common Vulnerabilities and Exposures (CVE)", + "url": "https://www.cve.org/CVERecord?id=CVE-2024-5555" + } + ], + "severity": "high", + "summary": "Imaginary ICS Controller provided by Example Industrial Corporation contains an authentication bypass vulnerability.", + "title": "Example vulnerability in Imaginary ICS Controller" +} diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml new file mode 100644 index 00000000..aa0278f3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml @@ -0,0 +1,53 @@ + + + + JVNDB Vulnerability countermeasure information + https://jvndb.jvn.jp/apis/myjvn + + 2024-03-10T01:05:00+09:00 + 2024-03-10T01:05:00+09:00 + + + + + + + + Example vulnerability in Imaginary ICS Controller + https://jvndb.jvn.jp/en/contents/2024/JVNDB-2024-123456.html + Sample advisory placeholder. + Information-technology Promotion Agency, Japan + 2024-03-10T01:00:00+09:00 + 2024-03-09T11:00:00+09:00 + 2024-03-10T01:00:00+09:00 + JVNDB-2024-123456 + + + diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml new file mode 100644 index 00000000..9c4b06e3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml @@ -0,0 +1,95 @@ + + + + JVNDB-2024-123456 + + Example vulnerability in Imaginary ICS Controller + + Imaginary ICS Controller provided by Example Industrial Corporation contains an authentication bypass vulnerability. + + + + Example Industrial Corporation + Imaginary ICS Controller firmware + cpe:2.3:o:example:imaginary_controller_firmware:2.0 + 2.0.5 + + + + + High + 8.8 + CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H + + + + + Apply firmware version 2.0.6 or later provided by the vendor. + + + + + Example ICS Vendor Advisory + EX-2024-01 + https://vendor.example.com/advisories/EX-2024-01 + + + Vendor advisory duplicate + https://vendor.example.com/advisories/EX-2024-01 + + + Common Vulnerabilities and Exposures (CVE) + CVE-2024-5555 + https://www.cve.org/CVERecord?id=CVE-2024-5555 + + + JVNDB + CWE-287 + Improper Authentication + https://cwe.mitre.org/data/definitions/287.html + + + + + 1 + 2024-03-09T11:00:00+09:00 + [2024/03/09] Initial advisory published. + + + 2 + 2024-03-10T11:30:00+09:00 + [2024/03/10] Vendor solution updated. + + + 2024-03-09T11:00:00+09:00 + 2024-03-10T11:30:00+09:00 + 2024-03-09T00:00:00+09:00 + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs new file mode 100644 index 00000000..c7a841cd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs @@ -0,0 +1,264 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Jvn; +using StellaOps.Feedser.Source.Jvn.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.JpFlags; +using Xunit.Abstractions; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Jvn.Tests; + +[Collection("mongo-fixture")] +public sealed class JvnConnectorTests : IAsyncLifetime +{ + private const string VulnId = "JVNDB-2024-123456"; + + private readonly MongoIntegrationFixture _fixture; + private readonly ITestOutputHelper _output; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public JvnConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 3, 10, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesDeterministicSnapshot() + { + var options = new JvnOptions + { + WindowSize = TimeSpan.FromDays(1), + WindowOverlap = TimeSpan.FromHours(6), + PageSize = 10, + }; + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + var now = _timeProvider.GetUtcNow(); + var windowStart = now - options.WindowSize; + var windowEnd = now; + + var overviewUri = BuildOverviewUri(options, windowStart, windowEnd, startItem: 1); + _handler.AddTextResponse(overviewUri, ReadFixture("jvnrss-window1.xml"), "application/xml"); + + var detailUri = BuildDetailUri(options, VulnId); + _handler.AddTextResponse(detailUri, ReadFixture("vuldef-JVNDB-2024-123456.xml"), "application/xml"); + + var connector = new JvnConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + + var stateAfterParse = await provider.GetRequiredService() + .TryGetAsync(JvnConnectorPlugin.SourceName, CancellationToken.None); + + await connector.MapAsync(provider, CancellationToken.None); + + var rawAdvisories = await _fixture.Database + .GetCollection("advisory") + .Find(Builders.Filter.Empty) + .ToListAsync(CancellationToken.None); + _output.WriteLine($"Fixture advisory count: {rawAdvisories.Count}"); + Assert.NotEmpty(rawAdvisories); + + var providerDatabase = provider.GetRequiredService(); + var providerCount = await providerDatabase + .GetCollection("advisory") + .CountDocumentsAsync(FilterDefinition.Empty, cancellationToken: CancellationToken.None); + _output.WriteLine($"Provider advisory count: {providerCount}"); + Assert.True(providerCount > 0, $"Provider DB advisory count was {providerCount}"); + + var typedDocs = await providerDatabase + .GetCollection("advisory") + .Find(FilterDefinition.Empty) + .ToListAsync(CancellationToken.None); + _output.WriteLine($"Typed advisory docs: {typedDocs.Count}"); + Assert.NotEmpty(typedDocs); + + var advisoryStore = provider.GetRequiredService(); + var singleAdvisory = await advisoryStore.FindAsync(VulnId, CancellationToken.None); + Assert.NotNull(singleAdvisory); + _output.WriteLine($"singleAdvisory null? {singleAdvisory is null}"); + + var canonical = SnapshotSerializer.ToSnapshot(singleAdvisory!); + var expected = ReadFixture("expected-advisory.json"); + Assert.Equal(expected, canonical); + + var jpFlagStore = provider.GetRequiredService(); + var jpFlag = await jpFlagStore.FindAsync(VulnId, CancellationToken.None); + Assert.NotNull(jpFlag); + Assert.Equal("product", jpFlag!.Category); + Assert.Equal("vulnerable", jpFlag.VendorStatus); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(JvnConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(JvnConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Empty(pendingDocs.AsBsonArray); + } + + private async Task EnsureServiceProviderAsync(JvnOptions template) + { + if (_serviceProvider is not null) + { + await ResetDatabaseAsync(); + return; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddJvnConnector(opts => + { + opts.BaseEndpoint = template.BaseEndpoint; + opts.WindowSize = template.WindowSize; + opts.WindowOverlap = template.WindowOverlap; + opts.PageSize = template.PageSize; + opts.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(JvnOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + _serviceProvider = services.BuildServiceProvider(); + var bootstrapper = _serviceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + } + + private Task ResetDatabaseAsync() + => _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + private static Uri BuildOverviewUri(JvnOptions options, DateTimeOffset windowStart, DateTimeOffset windowEnd, int startItem) + { + var (startYear, startMonth, startDay) = ToTokyoDateParts(windowStart); + var (endYear, endMonth, endDay) = ToTokyoDateParts(windowEnd); + + var parameters = new List> + { + new("method", "getVulnOverviewList"), + new("feed", "hnd"), + new("lang", "en"), + new("rangeDatePublished", "n"), + new("rangeDatePublic", "n"), + new("rangeDateFirstPublished", "n"), + new("dateFirstPublishedStartY", startYear), + new("dateFirstPublishedStartM", startMonth), + new("dateFirstPublishedStartD", startDay), + new("dateFirstPublishedEndY", endYear), + new("dateFirstPublishedEndM", endMonth), + new("dateFirstPublishedEndD", endDay), + new("startItem", startItem.ToString(CultureInfo.InvariantCulture)), + new("maxCountItem", options.PageSize.ToString(CultureInfo.InvariantCulture)), + }; + + return BuildUri(options.BaseEndpoint, parameters); + } + + private static Uri BuildDetailUri(JvnOptions options, string vulnId) + { + var parameters = new List> + { + new("method", "getVulnDetailInfo"), + new("feed", "hnd"), + new("lang", "en"), + new("vulnId", vulnId), + }; + + return BuildUri(options.BaseEndpoint, parameters); + } + + private static Uri BuildUri(Uri baseEndpoint, IEnumerable> parameters) + { + var query = string.Join( + "&", + parameters.Select(parameter => + $"{WebUtility.UrlEncode(parameter.Key)}={WebUtility.UrlEncode(parameter.Value)}")); + + var builder = new UriBuilder(baseEndpoint) + { + Query = query, + }; + + return builder.Uri; + } + + private static (string Year, string Month, string Day) ToTokyoDateParts(DateTimeOffset timestamp) + { + var local = timestamp.ToOffset(TimeSpan.FromHours(9)).Date; + return ( + local.Year.ToString("D4", CultureInfo.InvariantCulture), + local.Month.ToString("D2", CultureInfo.InvariantCulture), + local.Day.ToString("D2", CultureInfo.InvariantCulture)); + } + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Jvn", "Fixtures", filename); + return File.ReadAllText(path); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider?.Dispose(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj b/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj new file mode 100644 index 00000000..be7fa815 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/AGENTS.md b/src/StellaOps.Feedser.Source.Jvn/AGENTS.md new file mode 100644 index 00000000..e506a279 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/AGENTS.md @@ -0,0 +1,30 @@ +# AGENTS +## Role +Japan JVN/MyJVN connector; national CERT enrichment with strong identifiers (JVNDB) and vendor status; authoritative only where concrete package evidence exists; otherwise enriches text, severity, references, and aliases. +## Scope +- Fetch JVNRSS (overview) and VULDEF (detail) via MyJVN API; window by dateFirstPublished/dateLastUpdated; paginate; respect rate limits. +- Validate XML or JSON payloads; normalize titles, CVEs, JVNDB ids, vendor status, categories; map references and severity text; attach jp_flags. +- Persist raw docs with sha256 and headers; manage source_state cursor; idempotent parse/map. +## Participants +- Source.Common (HTTP, pagination, XML or XSD validators, retries/backoff). +- Storage.Mongo (document, dto, advisory, alias, affected (when concrete), reference, jp_flags, source_state). +- Models (canonical Advisory/Affected/Provenance). +- Core/WebService (jobs: source:jvn:fetch|parse|map). +- Merge engine applies enrichment precedence (does not override distro or PSIRT ranges unless JVN gives explicit package truth). +## Interfaces & contracts +- Aliases include JVNDB-YYYY-NNNNN and CVE ids; scheme "JVNDB". +- jp_flags: { jvndb_id, jvn_category, vendor_status }. +- References typed: advisory/vendor/bulletin; URLs normalized and deduped. +- Affected only when VULDEF gives concrete coordinates; otherwise omit. +- Provenance: method=parser; kind=api; value=endpoint plus query window; recordedAt=fetched time. +## In/Out of scope +In: JVN/MyJVN ingestion, aliases, jp_flags, enrichment mapping, watermarking. +Out: overriding distro or PSIRT ranges without concrete evidence; scraping unofficial mirrors. +## Observability & security expectations +- Metrics: jvn.fetch.requests, jvn.items, jvn.parse.fail, jvn.map.enriched_count, jvn.flags.jp_count. +- Logs: window bounds, jvndb ids processed, vendor_status distribution; redact API keys. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Jvn.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs b/src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs new file mode 100644 index 00000000..41706a67 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs @@ -0,0 +1,80 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Jvn.Configuration; + +/// +/// Options controlling the JVN connector fetch cadence and HTTP client configuration. +/// +public sealed class JvnOptions +{ + public static string HttpClientName => "source.jvn"; + + /// + /// Base endpoint for the MyJVN API. + /// + public Uri BaseEndpoint { get; set; } = new("https://jvndb.jvn.jp/myjvn", UriKind.Absolute); + + /// + /// Size of each fetch window applied to dateFirstPublished/dateLastUpdated queries. + /// + public TimeSpan WindowSize { get; set; } = TimeSpan.FromDays(7); + + /// + /// Overlap applied between consecutive windows to ensure late-arriving updates are captured. + /// + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(1); + + /// + /// Number of overview records requested per page (MyJVN max is 50). + /// + public int PageSize { get; set; } = 50; + + /// + /// Optional delay enforced between HTTP requests to respect service rate limits. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(500); + + /// + /// Maximum number of overview pages the connector will request in a single fetch cycle. + /// + public int MaxOverviewPagesPerFetch { get; set; } = 20; + + [MemberNotNull(nameof(BaseEndpoint))] + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("JVN options require an absolute BaseEndpoint."); + } + + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("WindowSize must be greater than zero."); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new InvalidOperationException("WindowOverlap cannot be negative."); + } + + if (WindowOverlap >= WindowSize) + { + throw new InvalidOperationException("WindowOverlap must be smaller than WindowSize."); + } + + if (PageSize is < 1 or > 50) + { + throw new InvalidOperationException("PageSize must be between 1 and 50 to satisfy MyJVN limits."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (MaxOverviewPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxOverviewPagesPerFetch must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs new file mode 100644 index 00000000..3020e37d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs @@ -0,0 +1,347 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.JpFlags; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal static class JvnAdvisoryMapper +{ + private static readonly string[] SeverityOrder = { "none", "low", "medium", "high", "critical" }; + + public static (Advisory Advisory, JpFlagRecord Flag) Map( + JvnDetailDto detail, + DocumentRecord document, + DtoRecord dtoRecord, + TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(detail); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + ArgumentNullException.ThrowIfNull(timeProvider); + + var recordedAt = dtoRecord.ValidatedAt; + var fetchProvenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt); + var mappingProvenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "mapping", detail.VulnerabilityId, recordedAt); + + var aliases = BuildAliases(detail); + var references = BuildReferences(detail, recordedAt); + var affectedPackages = BuildAffected(detail, recordedAt); + var cvssMetrics = BuildCvss(detail, recordedAt, out var severity); + + var description = DescriptionNormalizer.Normalize(new[] + { + new LocalizedText(detail.Overview, detail.Language), + }); + + var language = description.Language; + var summary = string.IsNullOrEmpty(description.Text) ? null : description.Text; + + var provenance = new[] { fetchProvenance, mappingProvenance }; + + var advisory = new Advisory( + detail.VulnerabilityId, + detail.Title, + summary, + language, + detail.DateFirstPublished, + detail.DateLastUpdated, + severity, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics, + provenance); + + var vendorStatus = detail.VendorStatuses.Length == 0 + ? null + : string.Join(",", detail.VendorStatuses.OrderBy(static status => status, StringComparer.Ordinal)); + + var flag = new JpFlagRecord( + detail.VulnerabilityId, + JvnConnectorPlugin.SourceName, + detail.JvnCategory, + vendorStatus, + timeProvider.GetUtcNow()); + + return (advisory, flag); + } + + private static IEnumerable BuildAliases(JvnDetailDto detail) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + detail.VulnerabilityId, + }; + + foreach (var cve in detail.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve); + } + } + + return aliases; + } + + private static IEnumerable BuildReferences(JvnDetailDto detail, DateTimeOffset recordedAt) + { + var references = new List(); + + foreach (var reference in detail.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + string? kind = reference.Type?.ToLowerInvariant() switch + { + "vendor" => "vendor", + "advisory" => "advisory", + "cwe" => "weakness", + _ => null, + }; + + string? sourceTag = !string.IsNullOrWhiteSpace(reference.Id) ? reference.Id : reference.Type; + string? summary = reference.Name; + + try + { + references.Add(new AdvisoryReference( + reference.Url, + kind, + sourceTag, + summary, + new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "reference", reference.Url, recordedAt))); + } + catch (ArgumentException) + { + // Ignore malformed URLs that slipped through validation. + } + } + + if (references.Count == 0) + { + return references; + } + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var reference in references) + { + if (!map.TryGetValue(reference.Url, out var existing)) + { + map[reference.Url] = reference; + continue; + } + + map[reference.Url] = MergeReferences(existing, reference); + } + + var deduped = map.Values.ToList(); + deduped.Sort(CompareReferences); + return deduped; + } + + private static IEnumerable BuildAffected(JvnDetailDto detail, DateTimeOffset recordedAt) + { + var packages = new List(); + + foreach (var product in detail.Affected) + { + if (string.IsNullOrWhiteSpace(product.Cpe)) + { + continue; + } + + if (!string.IsNullOrWhiteSpace(product.Status) && !product.Status.StartsWith("vulnerable", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!IdentifierNormalizer.TryNormalizeCpe(product.Cpe, out var cpe)) + { + continue; + } + + var provenance = new[] + { + new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "affected", cpe!, recordedAt) + }; + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Cpe, + cpe!, + platform: product.Vendor, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: provenance)); + } + + return packages; + } + + private static IReadOnlyList BuildCvss(JvnDetailDto detail, DateTimeOffset recordedAt, out string? severity) + { + var metrics = new List(); + severity = null; + var bestRank = -1; + + foreach (var cvss in detail.Cvss) + { + if (!CvssMetricNormalizer.TryNormalize(cvss.Version, cvss.Vector, cvss.Score, cvss.Severity, out var normalized)) + { + continue; + } + + var provenance = new AdvisoryProvenance(JvnConnectorPlugin.SourceName, "cvss", cvss.Type, recordedAt); + metrics.Add(normalized.ToModel(provenance)); + + var rank = Array.IndexOf(SeverityOrder, normalized.BaseSeverity); + if (rank > bestRank) + { + bestRank = rank; + severity = normalized.BaseSeverity; + } + } + + return metrics; + } + + private static int CompareReferences(AdvisoryReference? left, AdvisoryReference? right) + { + if (ReferenceEquals(left, right)) + { + return 0; + } + + if (left is null) + { + return 1; + } + + if (right is null) + { + return -1; + } + + var compare = StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.Kind, right.Kind); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.SourceTag, right.SourceTag); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.Summary, right.Summary); + if (compare != 0) + { + return compare; + } + + compare = StringComparer.Ordinal.Compare(left.Provenance.Source, right.Provenance.Source); + if (compare != 0) + { + return compare; + } + + compare = StringComparer.Ordinal.Compare(left.Provenance.Kind, right.Provenance.Kind); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.Provenance.Value, right.Provenance.Value); + if (compare != 0) + { + return compare; + } + + return left.Provenance.RecordedAt.CompareTo(right.Provenance.RecordedAt); + } + + private static int CompareNullable(string? left, string? right) + { + if (left is null && right is null) + { + return 0; + } + + if (left is null) + { + return 1; + } + + if (right is null) + { + return -1; + } + + return StringComparer.Ordinal.Compare(left, right); + } + + private static AdvisoryReference MergeReferences(AdvisoryReference existing, AdvisoryReference candidate) + { + var kind = existing.Kind ?? candidate.Kind; + var sourceTag = existing.SourceTag ?? candidate.SourceTag; + var summary = ChoosePreferredSummary(existing.Summary, candidate.Summary); + var provenance = existing.Provenance.RecordedAt <= candidate.Provenance.RecordedAt + ? existing.Provenance + : candidate.Provenance; + + if (kind == existing.Kind + && sourceTag == existing.SourceTag + && summary == existing.Summary + && provenance == existing.Provenance) + { + return existing; + } + + if (kind == candidate.Kind + && sourceTag == candidate.SourceTag + && summary == candidate.Summary + && provenance == candidate.Provenance) + { + return candidate; + } + + return new AdvisoryReference(existing.Url, kind, sourceTag, summary, provenance); + } + + private static string? ChoosePreferredSummary(string? left, string? right) + { + var leftValue = string.IsNullOrWhiteSpace(left) ? null : left; + var rightValue = string.IsNullOrWhiteSpace(right) ? null : right; + + if (leftValue is null) + { + return rightValue; + } + + if (rightValue is null) + { + return leftValue; + } + + return leftValue.Length >= rightValue.Length ? leftValue : rightValue; + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs new file mode 100644 index 00000000..91cdf8a2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal static class JvnConstants +{ + public const string DtoSchemaVersion = "jvn.vuldef.3.2"; + + public const string VuldefNamespace = "http://jvn.jp/vuldef/"; + public const string StatusNamespace = "http://jvndb.jvn.jp/myjvn/Status"; + public const string ModSecNamespace = "http://jvn.jp/rss/mod_sec/3.0/"; +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs new file mode 100644 index 00000000..e81c0b43 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs @@ -0,0 +1,106 @@ +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed record JvnCursor( + DateTimeOffset? WindowStart, + DateTimeOffset? WindowEnd, + DateTimeOffset? LastCompletedWindowEnd, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static JvnCursor Empty { get; } = new(null, null, null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + + if (WindowStart.HasValue) + { + document["windowStart"] = WindowStart.Value.UtcDateTime; + } + + if (WindowEnd.HasValue) + { + document["windowEnd"] = WindowEnd.Value.UtcDateTime; + } + + if (LastCompletedWindowEnd.HasValue) + { + document["lastCompletedWindowEnd"] = LastCompletedWindowEnd.Value.UtcDateTime; + } + + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())); + return document; + } + + public static JvnCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? windowStart = TryGetDateTime(document, "windowStart"); + DateTimeOffset? windowEnd = TryGetDateTime(document, "windowEnd"); + DateTimeOffset? lastCompleted = TryGetDateTime(document, "lastCompletedWindowEnd"); + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new JvnCursor(windowStart, windowEnd, lastCompleted, pendingDocuments, pendingMappings); + } + + public JvnCursor WithWindow(DateTimeOffset start, DateTimeOffset end) + => this with { WindowStart = start, WindowEnd = end }; + + public JvnCursor WithCompletedWindow(DateTimeOffset end) + => this with { LastCompletedWindowEnd = end }; + + public JvnCursor WithPendingDocuments(IEnumerable pending) + => this with { PendingDocuments = pending?.Distinct().ToArray() ?? Array.Empty() }; + + public JvnCursor WithPendingMappings(IEnumerable pending) + => this with { PendingMappings = pending?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? TryGetDateTime(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value)) + { + return null; + } + + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (element.BsonType == BsonType.String && Guid.TryParse(element.AsString, out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs new file mode 100644 index 00000000..07b369e2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs @@ -0,0 +1,67 @@ +using System.Collections.Immutable; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed record JvnDetailDto( + string VulnerabilityId, + string Title, + string? Overview, + string? Language, + DateTimeOffset? DateFirstPublished, + DateTimeOffset? DateLastUpdated, + DateTimeOffset? DatePublic, + ImmutableArray Cvss, + ImmutableArray Affected, + ImmutableArray References, + ImmutableArray History, + ImmutableArray CweIds, + ImmutableArray CveIds, + string? AdvisoryUrl, + string? JvnCategory, + ImmutableArray VendorStatuses) +{ + public static JvnDetailDto Empty { get; } = new( + "unknown", + "unknown", + null, + null, + null, + null, + null, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + null, + null, + ImmutableArray.Empty); +} + +internal sealed record JvnCvssDto( + string Version, + string Type, + string Severity, + double Score, + string? Vector); + +internal sealed record JvnAffectedProductDto( + string? Vendor, + string? Product, + string? Cpe, + string? Version, + string? Build, + string? Description, + string? Status); + +internal sealed record JvnReferenceDto( + string Type, + string Id, + string? Name, + string Url); + +internal sealed record JvnHistoryEntryDto( + string? Number, + DateTimeOffset? Timestamp, + string? Description); diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs new file mode 100644 index 00000000..2b3ee4b5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs @@ -0,0 +1,296 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Xml; +using System.Xml.Linq; +using System.Xml.Schema; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal static class JvnDetailParser +{ + private static readonly XNamespace Vuldef = JvnConstants.VuldefNamespace; + private static readonly XNamespace Status = JvnConstants.StatusNamespace; + + public static JvnDetailDto Parse(byte[] payload, string? documentUri) + { + ArgumentNullException.ThrowIfNull(payload); + + using var stream = new MemoryStream(payload, writable: false); + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Prohibit, + IgnoreComments = true, + IgnoreProcessingInstructions = true, + IgnoreWhitespace = true, + }; + + using var reader = XmlReader.Create(stream, settings); + var document = XDocument.Load(reader, LoadOptions.None); + Validate(document, documentUri); + return Extract(document, documentUri); + } + + private static void Validate(XDocument document, string? documentUri) + { + static bool IsToleratedValidationMessage(string? message) + { + if (string.IsNullOrWhiteSpace(message)) + { + return false; + } + + if (message.Contains("The 'vendor' attribute is not declared", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + + void Handler(object? sender, ValidationEventArgs args) + { + if (IsToleratedValidationMessage(args.Message)) + { + return; + } + + throw new JvnSchemaValidationException( + $"JVN schema validation failed for {documentUri ?? ""}: {args.Message}", + args.Exception ?? new XmlSchemaValidationException(args.Message)); + } + + try + { + document.Validate(JvnSchemaProvider.SchemaSet, Handler, addSchemaInfo: true); + } + catch (XmlSchemaValidationException ex) + { + if (!IsToleratedValidationMessage(ex.Message)) + { + throw new JvnSchemaValidationException( + $"JVN schema validation failed for {documentUri ?? ""}: {ex.Message}", ex); + } + } + } + + private static JvnDetailDto Extract(XDocument document, string? documentUri) + { + var root = document.Root ?? throw new InvalidOperationException("JVN VULDEF document missing root element."); + + var vulinfo = root.Element(Vuldef + "Vulinfo") ?? throw new InvalidOperationException("Vulinfo element missing."); + var vulinfoId = Clean(vulinfo.Element(Vuldef + "VulinfoID")?.Value) + ?? throw new InvalidOperationException("VulinfoID element missing."); + + var data = vulinfo.Element(Vuldef + "VulinfoData") ?? throw new InvalidOperationException("VulinfoData element missing."); + var title = Clean(data.Element(Vuldef + "Title")?.Value) ?? vulinfoId; + var overview = Clean(data.Element(Vuldef + "VulinfoDescription")?.Element(Vuldef + "Overview")?.Value); + + var dateFirstPublished = ParseDate(data.Element(Vuldef + "DateFirstPublished")?.Value); + var dateLastUpdated = ParseDate(data.Element(Vuldef + "DateLastUpdated")?.Value); + var datePublic = ParseDate(data.Element(Vuldef + "DatePublic")?.Value); + + var cvssEntries = ParseCvss(data.Element(Vuldef + "Impact")); + var affected = ParseAffected(data.Element(Vuldef + "Affected")); + var references = ParseReferences(data.Element(Vuldef + "Related")); + var history = ParseHistory(data.Element(Vuldef + "History")); + + var cweIds = references.Where(r => string.Equals(r.Type, "cwe", StringComparison.OrdinalIgnoreCase)) + .Select(r => r.Id) + .Where(static id => !string.IsNullOrWhiteSpace(id)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Select(static id => id!) + .ToImmutableArray(); + + var cveIds = references.Where(r => string.Equals(r.Type, "advisory", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(r.Id) + && r.Id.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) + .Select(r => r.Id) + .Where(static id => !string.IsNullOrWhiteSpace(id)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Select(static id => id!) + .ToImmutableArray(); + + var language = Clean(root.Attribute(XNamespace.Xml + "lang")?.Value); + + var statusElement = root.Element(Status + "Status"); + var jvnCategory = Clean(statusElement?.Attribute("category")?.Value); + + var vendorStatuses = affected + .Select(a => a.Status) + .Where(static status => !string.IsNullOrWhiteSpace(status)) + .Select(static status => status!.ToLowerInvariant()) + .Distinct(StringComparer.Ordinal) + .ToImmutableArray(); + + return new JvnDetailDto( + vulinfoId, + title, + overview, + language, + dateFirstPublished, + dateLastUpdated, + datePublic, + cvssEntries, + affected, + references, + history, + cweIds, + cveIds, + Clean(documentUri), + jvnCategory, + vendorStatuses); + } + + private static ImmutableArray ParseCvss(XElement? impactElement) + { + if (impactElement is null) + { + return ImmutableArray.Empty; + } + + var results = new List(); + foreach (var cvssElement in impactElement.Elements(Vuldef + "Cvss")) + { + var version = Clean(cvssElement.Attribute("version")?.Value) ?? ""; + var severityElement = cvssElement.Element(Vuldef + "Severity"); + var severity = Clean(severityElement?.Value) ?? Clean(cvssElement.Attribute("severity")?.Value) ?? string.Empty; + var type = Clean(severityElement?.Attribute("type")?.Value) ?? Clean(cvssElement.Attribute("type")?.Value) ?? "base"; + var scoreText = Clean(cvssElement.Element(Vuldef + "Base")?.Value) + ?? Clean(cvssElement.Attribute("score")?.Value) + ?? "0"; + if (!double.TryParse(scoreText, NumberStyles.Float, CultureInfo.InvariantCulture, out var score)) + { + score = 0d; + } + + var vector = Clean(cvssElement.Element(Vuldef + "Vector")?.Value) + ?? Clean(cvssElement.Attribute("vector")?.Value); + + results.Add(new JvnCvssDto( + version, + type, + severity, + score, + vector)); + } + + return results.ToImmutableArray(); + } + + private static ImmutableArray ParseAffected(XElement? affectedElement) + { + if (affectedElement is null) + { + return ImmutableArray.Empty; + } + + var results = new List(); + foreach (var item in affectedElement.Elements(Vuldef + "AffectedItem")) + { + var vendor = Clean(item.Element(Vuldef + "Name")?.Value); + var product = Clean(item.Element(Vuldef + "ProductName")?.Value); + var cpe = Clean(item.Element(Vuldef + "Cpe")?.Value); + var version = Clean(ReadConcatenated(item.Elements(Vuldef + "VersionNumber"))); + var build = Clean(ReadConcatenated(item.Elements(Vuldef + "BuildNumber"))); + var description = Clean(ReadConcatenated(item.Elements(Vuldef + "Description"))); + var status = Clean(item.Attribute("affectedstatus")?.Value); + + results.Add(new JvnAffectedProductDto(vendor, product, cpe, version, build, description, status)); + } + + return results.ToImmutableArray(); + } + + private static ImmutableArray ParseReferences(XElement? relatedElement) + { + if (relatedElement is null) + { + return ImmutableArray.Empty; + } + + var results = new List(); + foreach (var item in relatedElement.Elements(Vuldef + "RelatedItem")) + { + var type = Clean(item.Attribute("type")?.Value) ?? string.Empty; + var id = Clean(item.Element(Vuldef + "VulinfoID")?.Value) ?? string.Empty; + var name = Clean(item.Element(Vuldef + "Name")?.Value); + var url = Clean(item.Element(Vuldef + "URL")?.Value); + + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || (uri.Scheme is not "http" and not "https")) + { + continue; + } + + results.Add(new JvnReferenceDto(type, id, name, uri.ToString())); + } + + return results.ToImmutableArray(); + } + + private static ImmutableArray ParseHistory(XElement? historyElement) + { + if (historyElement is null) + { + return ImmutableArray.Empty; + } + + var results = new List(); + foreach (var item in historyElement.Elements(Vuldef + "HistoryItem")) + { + var number = Clean(item.Element(Vuldef + "HistoryNo")?.Value); + var timestamp = ParseDate(item.Element(Vuldef + "DateTime")?.Value); + var description = Clean(item.Element(Vuldef + "Description")?.Value); + results.Add(new JvnHistoryEntryDto(number, timestamp, description)); + } + + return results.ToImmutableArray(); + } + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } + + private static string? Clean(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim(); + } + + private static string? ReadConcatenated(IEnumerable elements) + { + var builder = new List(); + foreach (var element in elements) + { + var text = element?.Value; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + builder.Add(text.Trim()); + } + + return builder.Count == 0 ? null : string.Join("; ", builder); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs new file mode 100644 index 00000000..6272a7ec --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed record JvnOverviewItem( + string VulnerabilityId, + Uri DetailUri, + string Title, + DateTimeOffset? DateFirstPublished, + DateTimeOffset? DateLastUpdated); diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs new file mode 100644 index 00000000..2d712cea --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed record JvnOverviewPage( + IReadOnlyList Items, + int TotalResults, + int ReturnedCount, + int FirstResultIndex); diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs new file mode 100644 index 00000000..c5015ee3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Reflection; +using System.Threading; +using System.Xml; +using System.Xml.Schema; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal static class JvnSchemaProvider +{ + private static readonly Lazy<(XmlSchemaSet SchemaSet, EmbeddedResourceXmlResolver Resolver)> Cached = new( + LoadSchemas, + LazyThreadSafetyMode.ExecutionAndPublication); + + public static XmlSchemaSet SchemaSet => Cached.Value.SchemaSet; + + private static (XmlSchemaSet SchemaSet, EmbeddedResourceXmlResolver Resolver) LoadSchemas() + { + var assembly = typeof(JvnSchemaProvider).GetTypeInfo().Assembly; + var resourceMap = CreateResourceMap(); + var resolver = new EmbeddedResourceXmlResolver(assembly, resourceMap); + + var schemaSet = new XmlSchemaSet + { + XmlResolver = resolver, + }; + + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/vuldef_3.2.xsd"); + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/mod_sec_3.0.xsd"); + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/status_3.3.xsd"); + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/tlp_marking.xsd"); + AddSchema(schemaSet, resolver, "https://jvndb.jvn.jp/schema/data_marking.xsd"); + + schemaSet.Compile(); + return (schemaSet, resolver); + } + + private static void AddSchema(XmlSchemaSet set, EmbeddedResourceXmlResolver resolver, string uri) + { + using var stream = resolver.OpenStream(uri); + using var reader = XmlReader.Create(stream, new XmlReaderSettings { XmlResolver = resolver }, uri); + set.Add(null, reader); + } + + private static Dictionary CreateResourceMap() + { + var baseNamespace = typeof(JvnSchemaProvider).Namespace ?? "StellaOps.Feedser.Source.Jvn.Internal"; + var prefix = baseNamespace.Replace(".Internal", string.Empty, StringComparison.Ordinal); + + return new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["https://jvndb.jvn.jp/schema/vuldef_3.2.xsd"] = $"{prefix}.Schemas.vuldef_3.2.xsd", + ["vuldef_3.2.xsd"] = $"{prefix}.Schemas.vuldef_3.2.xsd", + ["https://jvndb.jvn.jp/schema/mod_sec_3.0.xsd"] = $"{prefix}.Schemas.mod_sec_3.0.xsd", + ["mod_sec_3.0.xsd"] = $"{prefix}.Schemas.mod_sec_3.0.xsd", + ["https://jvndb.jvn.jp/schema/status_3.3.xsd"] = $"{prefix}.Schemas.status_3.3.xsd", + ["status_3.3.xsd"] = $"{prefix}.Schemas.status_3.3.xsd", + ["https://jvndb.jvn.jp/schema/tlp_marking.xsd"] = $"{prefix}.Schemas.tlp_marking.xsd", + ["tlp_marking.xsd"] = $"{prefix}.Schemas.tlp_marking.xsd", + ["https://jvndb.jvn.jp/schema/data_marking.xsd"] = $"{prefix}.Schemas.data_marking.xsd", + ["data_marking.xsd"] = $"{prefix}.Schemas.data_marking.xsd", + ["https://www.w3.org/2001/xml.xsd"] = $"{prefix}.Schemas.xml.xsd", + ["xml.xsd"] = $"{prefix}.Schemas.xml.xsd", + }; + } + + private sealed class EmbeddedResourceXmlResolver : XmlResolver + { + private readonly Assembly _assembly; + private readonly Dictionary _resourceMap; + + public EmbeddedResourceXmlResolver(Assembly assembly, Dictionary resourceMap) + { + _assembly = assembly ?? throw new ArgumentNullException(nameof(assembly)); + _resourceMap = resourceMap ?? throw new ArgumentNullException(nameof(resourceMap)); + } + + public override ICredentials? Credentials + { + set { } + } + + public Stream OpenStream(string uriOrName) + { + var resourceName = ResolveResourceName(uriOrName) + ?? throw new FileNotFoundException($"Schema resource '{uriOrName}' not found in manifest."); + + var stream = _assembly.GetManifestResourceStream(resourceName); + if (stream is null) + { + throw new FileNotFoundException($"Embedded schema '{resourceName}' could not be opened."); + } + + return stream; + } + + public override object? GetEntity(Uri absoluteUri, string? role, Type? ofObjectToReturn) + { + if (absoluteUri is null) + { + throw new ArgumentNullException(nameof(absoluteUri)); + } + + var resourceName = ResolveResourceName(absoluteUri.AbsoluteUri) + ?? ResolveResourceName(absoluteUri.AbsolutePath.TrimStart('/')) + ?? ResolveResourceName(Path.GetFileName(absoluteUri.AbsolutePath)) + ?? throw new FileNotFoundException($"Schema resource for '{absoluteUri}' not found."); + + var stream = _assembly.GetManifestResourceStream(resourceName); + if (stream is null) + { + throw new FileNotFoundException($"Embedded schema '{resourceName}' could not be opened."); + } + + return stream; + } + + public override Uri ResolveUri(Uri? baseUri, string? relativeUri) + { + if (string.IsNullOrWhiteSpace(relativeUri)) + { + return base.ResolveUri(baseUri, relativeUri); + } + + if (Uri.TryCreate(relativeUri, UriKind.Absolute, out var absolute)) + { + return absolute; + } + + if (baseUri is not null && Uri.TryCreate(baseUri, relativeUri, out var combined)) + { + return combined; + } + + if (_resourceMap.ContainsKey(relativeUri)) + { + return new Uri($"embedded:///{relativeUri}", UriKind.Absolute); + } + + return base.ResolveUri(baseUri, relativeUri); + } + + private string? ResolveResourceName(string? key) + { + if (string.IsNullOrWhiteSpace(key)) + { + return null; + } + + if (_resourceMap.TryGetValue(key, out var resource)) + { + return resource; + } + + var fileName = Path.GetFileName(key); + if (!string.IsNullOrEmpty(fileName) && _resourceMap.TryGetValue(fileName, out resource)) + { + return resource; + } + + return null; + } + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs new file mode 100644 index 00000000..69ec80dc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs @@ -0,0 +1,16 @@ +using System; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +internal sealed class JvnSchemaValidationException : Exception +{ + public JvnSchemaValidationException(string message) + : base(message) + { + } + + public JvnSchemaValidationException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs b/src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs new file mode 100644 index 00000000..62f00787 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs @@ -0,0 +1,240 @@ +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Jvn.Configuration; + +namespace StellaOps.Feedser.Source.Jvn.Internal; + +public sealed class MyJvnClient +{ + private static readonly XNamespace RssNamespace = "http://purl.org/rss/1.0/"; + private static readonly XNamespace DcTermsNamespace = "http://purl.org/dc/terms/"; + private static readonly XNamespace SecNamespace = "http://jvn.jp/rss/mod_sec/3.0/"; + private static readonly XNamespace RdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; + private static readonly XNamespace StatusNamespace = "http://jvndb.jvn.jp/myjvn/Status"; + + private static readonly TimeSpan TokyoOffset = TimeSpan.FromHours(9); + + private readonly IHttpClientFactory _httpClientFactory; + private readonly JvnOptions _options; + private readonly ILogger _logger; + + public MyJvnClient(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + internal async Task> GetOverviewAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken) + { + if (windowEnd <= windowStart) + { + throw new ArgumentException("windowEnd must be greater than windowStart", nameof(windowEnd)); + } + + var items = new List(); + var client = _httpClientFactory.CreateClient(JvnOptions.HttpClientName); + + var startItem = 1; + var pagesFetched = 0; + + while (pagesFetched < _options.MaxOverviewPagesPerFetch) + { + cancellationToken.ThrowIfCancellationRequested(); + + var requestUri = BuildOverviewUri(windowStart, windowEnd, startItem); + using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var reader = XmlReader.Create(contentStream, new XmlReaderSettings { Async = true, IgnoreWhitespace = true, IgnoreComments = true }); + var document = await XDocument.LoadAsync(reader, LoadOptions.None, cancellationToken).ConfigureAwait(false); + + var page = ParseOverviewPage(document); + if (page.Items.Count == 0) + { + _logger.LogDebug("JVN overview page starting at {StartItem} returned zero results", startItem); + break; + } + + items.AddRange(page.Items); + pagesFetched++; + + if (page.ReturnedCount < _options.PageSize || startItem + _options.PageSize > page.TotalResults) + { + break; + } + + startItem += _options.PageSize; + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + return items; + } + + private Uri BuildOverviewUri(DateTimeOffset windowStart, DateTimeOffset windowEnd, int startItem) + { + var (startYear, startMonth, startDay) = ToTokyoDateParts(windowStart); + var (endYear, endMonth, endDay) = ToTokyoDateParts(windowEnd); + + var parameters = new[] + { + new KeyValuePair("method", "getVulnOverviewList"), + new KeyValuePair("feed", "hnd"), + new KeyValuePair("lang", "en"), + new KeyValuePair("rangeDatePublished", "n"), + new KeyValuePair("rangeDatePublic", "n"), + new KeyValuePair("rangeDateFirstPublished", "n"), + new KeyValuePair("dateFirstPublishedStartY", startYear), + new KeyValuePair("dateFirstPublishedStartM", startMonth), + new KeyValuePair("dateFirstPublishedStartD", startDay), + new KeyValuePair("dateFirstPublishedEndY", endYear), + new KeyValuePair("dateFirstPublishedEndM", endMonth), + new KeyValuePair("dateFirstPublishedEndD", endDay), + new KeyValuePair("startItem", startItem.ToString(CultureInfo.InvariantCulture)), + new KeyValuePair("maxCountItem", _options.PageSize.ToString(CultureInfo.InvariantCulture)), + }; + + var query = BuildQueryString(parameters); + + var builder = new UriBuilder(_options.BaseEndpoint) + { + Query = query, + }; + return builder.Uri; + } + + private static (string Year, string Month, string Day) ToTokyoDateParts(DateTimeOffset timestamp) + { + var local = timestamp.ToOffset(TokyoOffset).Date; + return ( + local.Year.ToString("D4", CultureInfo.InvariantCulture), + local.Month.ToString("D2", CultureInfo.InvariantCulture), + local.Day.ToString("D2", CultureInfo.InvariantCulture)); + } + + private static JvnOverviewPage ParseOverviewPage(XDocument document) + { + var items = new List(); + + foreach (var item in document.Descendants(RssNamespace + "item")) + { + var identifier = item.Element(SecNamespace + "identifier")?.Value?.Trim(); + if (string.IsNullOrWhiteSpace(identifier)) + { + continue; + } + + Uri? detailUri = null; + var linkValue = item.Element(RssNamespace + "link")?.Value?.Trim(); + if (!string.IsNullOrWhiteSpace(linkValue)) + { + Uri.TryCreate(linkValue, UriKind.Absolute, out detailUri); + } + + if (detailUri is null) + { + var aboutValue = item.Attribute(RdfNamespace + "about")?.Value?.Trim(); + if (!string.IsNullOrWhiteSpace(aboutValue)) + { + Uri.TryCreate(aboutValue, UriKind.Absolute, out detailUri); + } + } + + if (detailUri is null) + { + continue; + } + + var title = item.Element(RssNamespace + "title")?.Value?.Trim(); + if (string.IsNullOrWhiteSpace(title)) + { + title = identifier; + } + + var firstPublished = TryParseDate(item.Element(DcTermsNamespace + "issued")?.Value); + var lastUpdated = TryParseDate(item.Element(DcTermsNamespace + "modified")?.Value); + + items.Add(new JvnOverviewItem(identifier, detailUri, title!, firstPublished, lastUpdated)); + } + + var statusElement = document.Root?.Element(StatusNamespace + "Status") + ?? document.Descendants(StatusNamespace + "Status").FirstOrDefault(); + + var totalResults = TryParseInt(statusElement?.Attribute("totalRes")?.Value) ?? items.Count; + var returned = TryParseInt(statusElement?.Attribute("totalResRet")?.Value) ?? items.Count; + var firstResult = TryParseInt(statusElement?.Attribute("firstRes")?.Value) ?? 1; + + return new JvnOverviewPage(items, totalResults, returned, firstResult); + } + + private static DateTimeOffset? TryParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } + + private static int? TryParseInt(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : null; + } + + internal Uri BuildDetailUri(string vulnerabilityId) + { + ArgumentException.ThrowIfNullOrEmpty(vulnerabilityId); + + var query = BuildQueryString(new[] + { + new KeyValuePair("method", "getVulnDetailInfo"), + new KeyValuePair("feed", "hnd"), + new KeyValuePair("lang", "en"), + new KeyValuePair("vulnId", vulnerabilityId.Trim()), + }); + var builder = new UriBuilder(_options.BaseEndpoint) + { + Query = query, + }; + + return builder.Uri; + } + + private static string BuildQueryString(IEnumerable> parameters) + { + return string.Join( + "&", + parameters.Select(parameter => + $"{WebUtility.UrlEncode(parameter.Key)}={WebUtility.UrlEncode(parameter.Value)}")); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Jobs.cs b/src/StellaOps.Feedser.Source.Jvn/Jobs.cs new file mode 100644 index 00000000..2f06ed6a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Jvn; + +internal static class JvnJobKinds +{ + public const string Fetch = "source:jvn:fetch"; + public const string Parse = "source:jvn:parse"; + public const string Map = "source:jvn:map"; +} + +internal sealed class JvnFetchJob : IJob +{ + private readonly JvnConnector _connector; + + public JvnFetchJob(JvnConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class JvnParseJob : IJob +{ + private readonly JvnConnector _connector; + + public JvnParseJob(JvnConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class JvnMapJob : IJob +{ + private readonly JvnConnector _connector; + + public JvnMapJob(JvnConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs b/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs new file mode 100644 index 00000000..5172e66f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs @@ -0,0 +1,321 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Jvn.Configuration; +using StellaOps.Feedser.Source.Jvn.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.JpFlags; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Jvn; + +public sealed class JvnConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.General) + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly MyJvnClient _client; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IJpFlagStore _jpFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly TimeProvider _timeProvider; + private readonly JvnOptions _options; + private readonly ILogger _logger; + + public JvnConnector( + MyJvnClient client, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IJpFlagStore jpFlagStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _jpFlagStore = jpFlagStore ?? throw new ArgumentNullException(nameof(jpFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => JvnConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var windowEnd = now; + var defaultWindowStart = windowEnd - _options.WindowSize; + + var windowStart = cursor.LastCompletedWindowEnd.HasValue + ? cursor.LastCompletedWindowEnd.Value - _options.WindowOverlap + : defaultWindowStart; + + if (windowStart < defaultWindowStart) + { + windowStart = defaultWindowStart; + } + + if (windowStart >= windowEnd) + { + windowStart = windowEnd - TimeSpan.FromHours(1); + } + + _logger.LogInformation("JVN fetch window {WindowStart:o} - {WindowEnd:o}", windowStart, windowEnd); + + IReadOnlyList overviewItems; + try + { + overviewItems = await _client.GetOverviewAsync(windowStart, windowEnd, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to retrieve JVN overview between {Start:o} and {End:o}", windowStart, windowEnd); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + _logger.LogInformation("JVN overview returned {Count} items", overviewItems.Count); + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + + foreach (var item in overviewItems) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = _client.BuildDetailUri(item.VulnerabilityId); + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["jvn.vulnId"] = item.VulnerabilityId, + ["jvn.detailUrl"] = detailUri.ToString(), + }; + + if (item.DateFirstPublished.HasValue) + { + metadata["jvn.firstPublished"] = item.DateFirstPublished.Value.ToString("O"); + } + + if (item.DateLastUpdated.HasValue) + { + metadata["jvn.lastUpdated"] = item.DateLastUpdated.Value.ToString("O"); + } + + var result = await _fetchService.FetchAsync( + new SourceFetchRequest(JvnOptions.HttpClientName, SourceName, detailUri) + { + Metadata = metadata + }, + cancellationToken).ConfigureAwait(false); + + if (!result.IsSuccess || result.Document is null) + { + if (!result.IsNotModified) + { + _logger.LogWarning("JVN fetch for {Uri} returned status {Status}", detailUri, result.StatusCode); + } + + continue; + } + + _logger.LogDebug("JVN fetched document {DocumentId}", result.Document.Id); + pendingDocuments.Add(result.Document.Id); + } + + var updatedCursor = cursor + .WithWindow(windowStart, windowEnd) + .WithCompletedWindow(windowEnd) + .WithPendingDocuments(pendingDocuments); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + _logger.LogDebug("JVN parse pending documents: {PendingCount}", cursor.PendingDocuments.Count); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + _logger.LogDebug("JVN parsing document {DocumentId}", documentId); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + _logger.LogWarning("JVN document {DocumentId} no longer exists; skipping", documentId); + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("JVN document {DocumentId} is missing GridFS content; marking as failed", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to download raw JVN document {DocumentId}", document.Id); + throw; + } + + JvnDetailDto detail; + try + { + detail = JvnDetailParser.Parse(rawBytes, document.Uri); + } + catch (JvnSchemaValidationException ex) + { + _logger.LogWarning(ex, "JVN schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var sanitizedJson = JsonSerializer.Serialize(detail, SerializerOptions); + var payload = BsonDocument.Parse(sanitizedJson); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + JvnConstants.DtoSchemaVersion, + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + _logger.LogDebug("JVN parsed document {DocumentId}", documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + _logger.LogDebug("JVN map pending mappings: {PendingCount}", cursor.PendingMappings.Count); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + _logger.LogWarning("Skipping JVN mapping for {DocumentId}: DTO or document missing", documentId); + pendingMappings.Remove(documentId); + continue; + } + + var dtoJson = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + JvnDetailDto detail; + try + { + detail = JsonSerializer.Deserialize(dtoJson, SerializerOptions) + ?? throw new InvalidOperationException("Deserialized DTO was null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize JVN DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var (advisory, flag) = JvnAdvisoryMapper.Map(detail, document, dto, _timeProvider); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _jpFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + _logger.LogDebug("JVN mapped document {DocumentId}", documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? JvnCursor.Empty : JvnCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(JvnCursor cursor, CancellationToken cancellationToken) + { + var cursorDocument = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, cursorDocument, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs new file mode 100644 index 00000000..26f4ac06 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Jvn; + +public sealed class JvnConnectorPlugin : IConnectorPlugin +{ + public string Name => SourceName; + + public static string SourceName => "jvn"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs new file mode 100644 index 00000000..0627ac68 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Jvn.Configuration; + +namespace StellaOps.Feedser.Source.Jvn; + +public sealed class JvnDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:jvn"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddJvnConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, JvnJobKinds.Fetch, typeof(JvnFetchJob)); + EnsureJob(options, JvnJobKinds.Parse, typeof(JvnParseJob)); + EnsureJob(options, JvnJobKinds.Map, typeof(JvnMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs new file mode 100644 index 00000000..38275e08 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Jvn.Configuration; +using StellaOps.Feedser.Source.Jvn.Internal; + +namespace StellaOps.Feedser.Source.Jvn; + +public static class JvnServiceCollectionExtensions +{ + public static IServiceCollection AddJvnConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(JvnOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Jvn/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/xml"; + }); + + services.AddTransient(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd new file mode 100644 index 00000000..0a48bd17 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd @@ -0,0 +1,91 @@ + + + + This schema was originally developed by The MITRE Corporation. The Data Marking XML Schema implementation is maintained by The MITRE Corporation and developed by the open STIX Community. For more information, including how to get involved in the effort and how to submit change requests, please visit the STIX website at http://stix.mitre.org. + + Data Marking + 1.1.1 + 05/08/2014 9:00:00 AM + Data_Marking - Schematic implementation for an independent, flexible, structured data marking expression. + Copyright (c) 2012-2014, The MITRE Corporation. All rights reserved. The contents of this file are subject to the terms of the STIX License located at http://stix.mitre.org/about/termsofuse.html. See the STIX License for the specific language governing permissions and limitations for use of this schema. When distributing copies of the Data Marking Schema, this license header must be included. + + + + + MarkingType specifies a structure for marking information to be applied to portions of XML content. + + + + + This field contains specification of marking information to be applied to portions of XML content. + + + + + + + The MarkingStructureType contains the marking information to be applied to a portion of XML content. + This type is defined as abstract and is intended to be extended to enable the expression of any structured or unstructured data marking mechanism. The data marking structure is simply a mechanism for applying existing marking systems to nodes. The data marking systems themselves define the semantics of what the markings mean, how multiple markings to the same node should be applied, and what to do if a node is unmarked. + It is valid per this specification to mark a node with multiple markings from the same system or mark a node across multiple marking systems. If a node is marked multiple times using the same marking system, that system specifies the semantic meaning of multiple markings and (if necessary) how conflicts should be resolved. If a node is marked across multiple marking systems, each system is considered individually applicable. If there are conflicting markings across marking systems the behavior is undefined, therefore producers should make every effort to ensure documents are marked consistently and correctly among all marking systems. + STIX provides two marking system extensions: Simple, and TLP. Those who wish to use another format may do so by defining a new extension to this type. The STIX-provided extensions are: + 1. Simple: The Simple marking structure allows for the specification of unstructured statements through the use of a string field. The type is named SimpleMarkingStructureType and is in the http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1 namespace. The extension is defined in the file extensions/marking/simple_marking.xsd or at the URL http://stix.mitre.org/XMLSchema/extensions/marking/simple_marking/1.1.1/simple_marking.xsd. + 2. TLP: The TLP marking structure allows for the expression of Traffic Light Protocol statements through the use of a simple enumeration. The type is named TLPMarkingStructureType and is in the http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1 namespace. The extension is defined in the file extensions/marking/tlp_marking.xsd or at the URL http://stix.mitre.org/XMLSchema/extensions/marking/tlp/1.1.1/tlp_marking.xsd. + 3. Terms of Use: The Terms of Use marking structure allows for the specification of unstructured terms of use statements through the use of a string field. The type is named TermsOfUseMarkingStructureType and is in the http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1 namespace. The extension is defined in the file extensions/marking/terms_of_use_marking.xsd or at the URL http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.0.1/terms_of_use_marking.xsd. + + + + This field specifies the name of the marking model to be applied within this Marking_Structure. + + + + + This field contains a reference to an authoritative source on the marking model to be applied within this Marking_Structure. + + + + + Specifies a unique ID for this Marking_Structure. + + + + + Specifies a reference to the ID of a Marking_Structure defined elsewhere. + When idref is specified, the id attribute must not be specified, and any instance of this Marking_Structure should not hold content. + + + + + + + + This field utilizes XPath 1.0 to specify the structures for which the Marking is to be applied. + The XPath expression is NOT recursive and the marking structure does NOT apply to child nodes of the selected node. Instead, the expression must explicitly select all nodes that the marking is to be applied to including elements, attributes, and text nodes. + The context root of the XPath statement is this Controlled_Structure element. Any namespace prefix declarations that are available to this Controlled_Structure element are available to the XPath. + Note that all Controlled_Structure elements have a scope within the document for which their XPath is valid to reference. + Usages of MarkingType may specify a "marking scope". The marking scope is always recursive and specifies the set of nodes that may be selected by the XPath expression (and therefore that may have the markings applied to them). If no marking scope is specified in the schema documentation or specification where the MarkingType is used, it should be assumed that the document itself and all nodes are within scope. + + + + + This field contains the marking information to be applied to the portions of XML content specified in the ControlledStructure field. This field is defined as MarkingStructureType which is an abstract type the enables the flexibility to utilize any variety of marking structures. + + + + + + Specifies a unique ID for this Marking. + + + + + Specifies a reference to the ID of a Marking defined elsewhere. + When idref is specified, the id attribute must not be specified, and any instance of this Marking should not hold content. + + + + + Specifies the relevant Data_Marking schema version for this content. + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd new file mode 100644 index 00000000..858622f6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd @@ -0,0 +1,133 @@ + + + + + + + + + + + + + + + + + + + JVNRSS is based on RDF Site Summary (RSS) 1.0 and use the + field dc:relation of Dublin Core / sec:references of mod_sec as index of grouping + security information. + JVNRSS は、脆弱性対策情報の概要記述用 XML フォーマットで、サイトの概要をメタデータとして簡潔に記述する + XML フォーマットである RSS (RDF Site Summary) 1.0 をベースとした仕様です。他サイトに掲載可能な形式で発信する仕組み、脆弱性対策情報のグループ化 + (dc:relation, sec:references) + や抽出した情報の再構成などの点から、脆弱性対策情報の利活用を促進することを目的としています。 + https://jvndb.jvn.jp/en/schema/jvnrss.html + https://jvndb.jvn.jp/schema/jvnrss.html + + JVN RDF Site Summary (JVNRSS) + Masato Terada + 3.2 + 2017-07-20T03:16:00+09:00 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd new file mode 100644 index 00000000..676c85a3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + mod_sec describes RSS Extension of security information + distribution, and definition of the tags for RSS 1.0, 2.0 and Atom. + mod_sec は、脆弱性対策情報などのセキュリティ情報を記述するための JVNRSS 拡張仕様で、RSS + 1.0、RSS 2.0、Atom での利用を想定した汎用的な仕様となっています。 + https://jvndb.jvn.jp/en/schema/mod_sec.html + https://jvndb.jvn.jp/schema/mod_sec.html + + Qualified Security Advisory Reference (mod_sec) + Masato Terada + 3.0 + 2017-07-20T03:16:00+09:00 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Specifies the relevant handling guidance for this STIX_Package. The + valid marking scope is the nearest STIXPackageType ancestor of this Handling element + and all its descendants. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd new file mode 100644 index 00000000..5f189854 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd @@ -0,0 +1,574 @@ + + + + + + + + + + + + + This is an XML Schema for the status information of MyJVN API. + MyJVN API のステータス情報を格納する XML スキーマ + + Status Information of MyJVN API + Masato Terada + 3.3 + 2017-07-20T03:16:00+09:00 + + + + + + + + + + + + + + + + + + Response Parameter; MyJVN API Schema Version - MyJVN API Ver 3.0 [common] + レスポンスパラメタ; MyJVN API スキーマバージョン - MyJNV API Ver 3.0 [共通] + + + + + Response Parameter; Return Code/Interger (0:success, 1:failure) [common] + レスポンスパラメタ; リターンコード/整数値 (0:成功, 1:エラー) [共通] + + + + + Request Parameter; Maximum number of Entry/Interger [common] + リクエストパラメタ; エントリ上限値/整数値 (APIごとに規定されている一度に取得できるエントリ件数の上限値, エラー時は空文字列) [共通] + + + + + Response Parameter; Error Code (Null:success) [common] + レスポンスパラメタ; エラーコード (空文字列:成功) [共通] + + + + + Response Parameter; Error Message (Null:success) [common] + レスポンスパラメタ; エラーメッセージ (空文字列:成功) [共通] + + + + + Response Parameter; Total number of Result entries [common] + レスポンスパラメタ; 応答エントリ総数: 整数値 (フィルタリング条件に当てはまるエントリの総件数) ；エラー時は空文字列 [共通] + + + + + Response Parameter; Number of Result entries [common] + レスポンスパラメタ; 応答エントリ数: 整数値 (フィルタリング条件に当てはまるエントリのうち、レスポンスに格納されている件数) ；エラー時は空文字列 [共通] + + + + + Response Parameter; Start entry number in Result entries [common] + レスポンスパラメタ; 応答エントリ開始位置: 整数値 (フィルタリング条件に当てはまるエントリのうち、何番目からのデータを取得したのかを示す値) ；エラー時は空文字列 [共通] + + + + + + + + + Request Parameter; Method [common] + リクエストパラメタ; メソッド名 [共通] + + + + + Request Parameter; Language (ja/en) [common] + リクエストパラメタ; 表示言語 (ja/en) [共通] + + + + + Request Parameter: Start entry number [common] + リクエストパラメタ: エントリ開始位置 [共通] + + + + + Request Parameter: Read entry number [common] + リクエストパラメタ: エントリ取得件数 [共通] + + + + + Request Parameter: XSL file enable/disable [common] + リクエストパラメタ: XSL ファイル 適用／未適用 [共通] + + + + + Request Parameter: feed name + リクエストパラメタ: フェードフォーマット(=APIバージョン)を示す名称 + + + + + + + + + Request Parameter: Vendor CPE Name/Product CPE Name + リクエストパラメタ: ベンダ CPE 名/製品 CPE 名 + + + + + Request Parameter: Vendor unique numbers + リクエストパラメタ: ベンダの識別番号一覧 + + + + + Request Parameter: Product unique numbers + リクエストパラメタ: 製品の識別番号一覧 + + + + + Request Parameter: Keyword + リクエストパラメタ: キーワード + + + + + Request Parameter: Type of OVAL + リクエストパラメタ: OVAL 種別 + method=getOvalList, getVulnOverviewStatistics + + + + + Request Parameter: Type of feed limit + リクエストパラメタ: フィード制限タイプ + method=getVendorList, getProductList,getVulnOverviewList, getVulnDetailInfo + + + + + + + + + Request Parameter: Product type (01/02/03) + リクエストパラメタ: 製品タイプ (01/02/03) + method=getProductList + + + + + Request Parameter: MyJVN API Version + リクエストパラメタ: MyJVN API Version + method=getProductList + + + + + Response Paramter; ReLatest date of product registration/update + レスポンスパラメタ: 製品登録/更新の最新日 + method=getProductList + + + + + + + + + Request Parameter: Severity + リクエストパラメタ: CVSS 深刻度 + + + + + Request Parameter: Vector of CVSS Base metric + リクエストパラメタ: CVSS 基本評価基準ベクタ + + + + + Request Parameter: Range of Date Public + リクエストパラメタ: 発見日の範囲指定 + + + + + Request Parameter: Range of Date Last Updated + リクエストパラメタ: 更新日の範囲指定 + + + + + Request Parameter: Range of Date First Published + リクエストパラメタ: 発行日の範囲指定 + + + + + Request Parameter: Start year of Date Public + リクエストパラメタ: 発見日開始年 + method=getVulnOverviewList + method=getStatistics + + + + + Request Parameter: Start month of Date Public + リクエストパラメタ: 発見日開始月 + method=getVulnOverviewList + method=getStatistics + + + + + Request Parameter: Start day of Date Public + リクエストパラメタ: 発見日開始日 + + + + + Request Parameter: End year of Date Public + リクエストパラメタ: 発見日終了年 + method=getVulnOverviewList + method=getStatistics + + + + + Request Parameter: End month of Date Public + リクエストパラメタ: 発見日終了月 + method=getVulnOverviewList + method=getStatistics + + + + + Request Parameter: End day of Date Public + リクエストパラメタ: 発見日終了日 + + + + + Request Parameter: Start year of Date Last Updated + リクエストパラメタ: 更新日開始年 + + + + + Request Parameter: Star month of Date Last Updated + リクエストパラメタ: 更新日開始月 + + + + + Request Parameter: Start day of Date Last Updated + リクエストパラメタ: 更新日開始日 + + + + + Request Parameter: End year of Date Last Updated + リクエストパラメタ: 更新日終了年 + + + + + Request Parameter: End month of Date Last Updated + リクエストパラメタ: 更新日終了月 + + + + + Request Parameter: End day of Date Last Updated + リクエストパラメタ: 更新日終了日 + + + + + Request Parameter: Start year of Date First Published + リクエストパラメタ: 発行日開始年 + + + + + Request Parameter: Start month of Date First Published + リクエストパラメタ: 発行日開始月 + + + + + Request Parameter: Start day of Date First Published + リクエストパラメタ: 発行日開始日 + + + + + Request Parameter: End year of Date First Published + リクエストパラメタ: 発行日終了年 + + + + + Request Parameter: End month of Date First Published + リクエストパラメタ: 発行日終了月 + + + + + Request Parameter: End day of Date First Published + リクエストパラメタ: 発行日終了日 + + + + + + + + + Request Parameter: Vulnerability ID + リクエストパラメタ: 脆弱性対策情報 ID + method=getVulnDetailInfo + + + + + + + + + Request Parameter: Vulnerability ID + リクエストパラメタ: 脆弱性対策情報 ID + method=getCvrfInfo + + + + + + + + + Request Parameter: Type of OS + リクエストパラメタ: OS 種別 + method=getOvalList + + + + + Request Parameter: Type of OVAL definition + リクエストパラメタ: OVAL定義のタイプ + method=getOvalList + + + + + Request Parameter: Type of Application condition + リクエストパラメタ: アプリケーションの動作モード + method=getOvalList + + + + + + + + + Request Parameter: OVAL ID + リクエストパラメタ: OVAL ID + method=getOvalData + + + + + + + + + Request Parameter: Benchmark ID + リクエストパラメタ: ベンチマーク ID + method=getXccdfCheckData + + + + + + + + + Request Parameter: Graph theme + リクエストパラメタ: グラフ テーマ + method=getStatistics + + + + + Response Parameter: Maxium number of cntAll + レスポンスパラメタ: cntAll の最大値 + method=getStatistics + + + + + Request Parameter: CWE ID + リクエストパラメタ: CWE 識別子 + method=getStatistics + + + + + Request Parameter: Product unique numbers + リクエストパラメタ: 製品の識別番号一覧 + + + + + + + + Request Parameter: reference + リクエストパラメタ: 参考情報 + method=getCPEdictionary + + + + + + + + Request Parameter: Date Last Updated (Year 4digits) + リクエストパラメタ: 更新日年 + method=getAlertList + + + + + + Request Parameter: Date First Published (Year 4digits) + リクエストパラメタ: 発行日年 + method=getAlertList + + + + + + Request Parameter: reference + リクエストパラメタ: 参考情報 + method=getAlertList + + + + + + + + + + + Define the version Number of Status XSD + Status XSD のバージョン番号 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd new file mode 100644 index 00000000..9b728aaf --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd @@ -0,0 +1,40 @@ + + + + This schema was originally developed by The MITRE Corporation. The Data Marking Schema implementation is maintained by The MITRE Corporation and developed by the open STIX Community. For more information, including how to get involved in the effort and how to submit change requests, please visit the STIX website at http://stix.mitre.org. + + Data Marking Extension - TLP + 1.1.1 + 05/08/2014 9:00:00 AM + Data Marking Extension - TLP Marking Instance - Schematic implementation for attaching a Traffic Light Protocol (TLP)designation to an idendified XML structure. + Copyright (c) 2012-2014, The MITRE Corporation. All rights reserved. The contents of this file are subject to the terms of the STIX License located at http://stix.mitre.org/about/termsofuse.html. See the STIX License for the specific language governing permissions and limitations for use of this schema. When distributing copies of the STIX Schema, this license header must be included. + + + + + + The TLPMarkingStructureType is an implementation of the data marking schema that allows for a TLP Designation to be attached to an identified XML structure. Information about TLP is available here: http://www.us-cert.gov/tlp. + Nodes may be marked by multiple TLP Marking statements. When this occurs, the node should be considered marked at the most restrictive TLP Marking of all TLP Markings that were applied to it. For example, if a node is marked both GREEN and AMBER, the node should be considered AMBER. + + + + + + The TLP color designation of the marked structure. + + + + + + + + The TLP color designation of the marked structure. + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd new file mode 100644 index 00000000..fde22c33 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd @@ -0,0 +1,1564 @@ + + + + + + + + + + + + + This is an XML Schema for VULDEF - The Vulnerability Data + Publication and Exchange Format Data Model. + 脆弱性詳細情報の XML スキーマ VULDEF - The Vulnerability Data + Publication and Exchange Format Data Model + + VULDEF - The Vulnerability Data Publication and Exchange Format Data + Model + Masato Terada + 3.2 + 2017-07-20T03:16:00+09:00 + + + + + "VULDEF(The VULnerability Data publication and Exchange + Format data model)" is intended to be a format for the security information published by + the vendors or the Computer Security Incident Response Teams (CSIRTs). Assuming + widespread adoption of the VULDEF by the community, an organization can potentially + benefit from the increased automation in the processing of security advisory data, since + the commitment of vulnerability handling to parse free-form textual document will be + reduced. + "VULDEF(The VULnerability Data publication and Exchange + Format data model)" + の目的は、脆弱性情報ならびに脆弱性を除去するための脆弱性対策情報を提供し、流通させるために必要となるデータフォーマットを定義することにある。特に、脆弱性対策情報については、データフォーマットを定義することにより、情報自身の流通ならびに、関連対策情報同士の集約化を促すことができ、結果として対策促進を支援することができるであろう。 + + + The purpose of the "VULDEF(The VULnerability Data + publication and Exchange Format data model)" is to define data formats for information + related to security advisory typically published by the Vendors and Computer Security + Incident Response Teams (CSIRTs). An the Extensible Markup Language (XML) Document Type + Definition is developed, and examples are provided. + "VULDEF(The VULnerability Data publication and Exchange + Format data model)" では、脆弱性対策の情報提供(含む交換)において必要となる項目をデータモデルとして提示すると共に、XML + による表現形式を規定することにある。 + + + + + + + + + + + + VULDEF-Document class is the top level class in the + VULDEF data model and the DTD. All VULDEF documents are instances of the + VULDEF-Document class. The version of the VULDEF specification to which the VULDEF + document conforms. The value of this attribute MUST be 3.2. + VULDEFドキュメントクラスは、VULDEF データモデルと DTD のトップレベルのクラスである。全ての + VULDEF ドキュメントは、VULDEF ドキュメントクラスのインスタンスとなる。VULDEF のバージョン情報には "3.2" + を設定する。 + + + + + + + + + + + + + + + + + + + + In each publication of vulnerability related data is + represented by an instance of the Vulinfo class. This class provides a standardized + representation for commonly published vulnerability data and associates a unique + identifier. + Vulinfo + クラスは、脆弱性に関する情報(概要、想定される影響、対策など)を記載するクラスと、その脆弱性情報を一意に識別する識別子クラスから構成する。 + + + + + + + + + + + + + + + + + + VulinfoID class represents an vulnerability + information number that is unique in the context of the vendor or CSIRT and + identifies the activity characterized in an VULDEF-Document. A vulnerability number + assigned to this vulnerability information by the party that generated the document. + VulinfoID includes the organization prefix and unique number within the + organization. ex. {TA04-217A:US-CERT Alerts (CERT-TA)}{bid9835:Bugtraq + (BID)}{XF9324:ISS X-Force (XF)}{JVN54326:VN-JP (JVN)} + 脆弱性情報を一意に識別するための識別子であり、脆弱性情報を作成した組織が割り当てる。 + + + + + + + + + Group ID for vulnerability + information + 複数の脆弱性情報を取り扱う場合のグループ識別子を記載する。 + + + + + + + + + + + + + + + + + + + + + + The item(s) that constitute the vulnerability about + which the VULDEF-Document conveys information. The VulinfoData class summarizes the + details of the vulnerability information. + VulinfoData + クラスは、脆弱性情報として、脆弱性の概要、想定される影響、対策などの情報を記載する。 + + + + + + Title class describes the title of the + vulnerability information. + 脆弱性対策情報の題名を記載する。JVNRSS の item 要素の title + に対応する。 + + + + + + + + + + + + + + + + + + + + + + + + + + + VulinfoDescription class summarizes the detail of the + vulnerability information. + VulinfoDescription + クラスは、脆弱性に関する概要、技術的な解説、脆弱性のタイプの情報を記載する。 + + + + + + + + + + + + + + + + + + + + Overview is an abstract of the vulnerability that + provides a summary of the problem and its impact to the reader. + 脆弱性ならびにその対策に関する概要を記載する。JVNRSS1.0 の item 要素の description + に対応する。 + + + + + + + + + + + + + + + + + The vulnerability description contains one or more + paragraphs of text describing the vulnerability. + 脆弱性に関する詳細情報(技術的な解説)を記載する。 + + + + + + + + + + + + + + CWE + 脆弱性に関するタイプを記載する。 + + + + + + + + + + + + + + + + + Affected class includes vendors who may be affected + by the vulnerability. + Affected + クラスは、脆弱性により影響を受けるバージョン、システムに関する情報を提示するクラスである。 + + + + + + + + + + + + Entries in the Affected class. + 影響を受ける製品の項目 + + + + + + A vendor name of the affected + products. + 影響を受ける製品のベンダ名(提供者名)を記載する。 + + + + + + + + + A free-form textual description of the + affected products. + 影響を受ける製品に関する説明 + + + + + + + + + + + + A product name of the affected products. + 影響を受ける製品名を記載する。 + + + + + + + + + + + + + A version number of the affected products. + 影響を受ける製品のバージョンあるいはリビジョン番号を記載する。 + + + + + + + + + + + + + + A build number of the affected products. + 影響を受ける製品のビルド番号を記載する。 + + + + + + + + + + + + + + A version or build number of the affected products. + 影響を受ける製品のバージョン番号あるいはビルド番号の範囲を記載する。 + + + + + + + + + + + + + A version or revision number of the affected + products. + 影響を受ける製品のバージョン番号あるいはビルド番号の範囲を記載する。 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Impact class allows for classifying as well as + providing a description of the technical impact due to the + vulnerability. + Impact クラスは、脆弱性に伴い想定しうる影響を記載するクラスである。 + + + + + + + + + + + + + Cvss class is a information of the Common + Vulnerability Scoring System. + CVSS に関する情報を記載するクラスである。 + + + + + + + + + + + + + + + + + CVSS severity ranking. + CVSS 深刻度 + + + + + + + + + + + + + + CVSS Vector Strings. + CVSS 短縮表記 + + + + + + + + + + + + + CVSS Base Score. + CVSS 基本値 + + + + + + + + + + + + + CVSS Temporal Score. + CVSS 現状値 + + + + + + + + + + + + + CVSS Environmental Score. + CVSS 環境値 + + + + + + + + + + + + + Entries in the Impact class. + 想定される影響の項目 + + + + + + A free-form textual description of the + impact. + 想定される影響の項目に関する説明 + + + + + + + + + + + + + + + + + Solution class allows for classifying as well as + providing a description of the technical solution due to the + vulnerability. + Solution + クラスは、脆弱性の回避施策に関する情報を記載するクラスである。 + + + + + + + + + + + + Entries in the Solution class. + 脆弱性の回避施策の項目 + + + + + + A free-form textual description of the + solution. + 脆弱性の回避施策に関する説明 + + + + + + + + + + + + + + + + + Exploit class allows for classifying as well as + providing a description of the technical exploit due to the vulnerability. + Exploitクラスは、脆弱性の攻略に関する情報を記載するクラスである。 + + + + + + + + + + + + Entries in the Exploit class. + 脆弱性の攻略に関する項目 + + + + + + A free-form textual description of the + exploit. + 脆弱性の攻略に関する説明 + + + + + A URL to additional information about the + exploit. + 脆弱性の攻略に関する情報掲載 URL + + + + + + + + + + + + + + + + Related class is a collection of URLs at our web site + and others providing additional information about the vulnerability. + Relatedクラスは、参考情報など脆弱性に関連する情報を記載するクラスである。 + + + + + + + + + + + + Entries in the Related class. + 関連情報の項目を記載する。 + + + + + + A issuer of the + reference. + 脆弱性対策情報発行者の名称 + + + + + A ID of the reference. + 脆弱性対策情報を一意に識別するための識別子 + + + + + A title of the reference. + 脆弱性対策情報の題名 + + + + + A URL to related information about the + vulnerability. + 脆弱性対策情報の掲載 URL。JVNRSS の item 要素の dc:relation + に対応付ける。 + + + + + A free-form textual description of the + reference. + 関連情報の項目に関する説明 + + + + + + + + + + + + + + + + + Credit Class identifies who initially discovered the + vulnerability, anyone who was instrumental in the development of the document and + the contributors for anything. + + + + + + + + + + + + + Entries in the Credit class. + + + + + + + An author/contributor Name. + + + + + + A free-form textual description of the + credit. + + + + + + + + + + + + + + + + Contact class describes contact information of + VULDEF-Document issuer. + + + + + + + + + + + + + Entries in the Contact class. + + + + + + + + + + + + + + + + + + + + + + + + History class is a log or diary of the significant + events that occurred or actions performed by the issuers. + History クラスは、脆弱性情報の改訂履歴などを記載するクラスである。 + + + + + + + + + + + + HistoryItem class is a particular entry in the + History log that documents a particular significant action or + event. + 改訂履歴の項目 + + + + + + + + + + + + + + + + Number of the this entry in the history + log. + 改訂履歴の項目に付与する番号 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + simpleType used when scoring on a scale of 0-10, + inclusive. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This attribute indicates the disclosure guidelines to + which the sender expects the recipient of the VULDEF-Document to adhere. This + attribute is defined as an enumerated value with a default value of + "private". + 送信側がVULDEF-Documentの受信側に期待する配布のガイドライン属性であり、以下の属性値(デフォルト値＝private)を選択する。 + + + + + There is no restriction level applied to the + information. + 情報配布に関する制約はない。 + + + + + The information may not be + shared. + 共有を期待する情報ではない。 + + + + + + + + The historyno attribute refers to HistoryNo class. + 改訂履歴の項目に付与する番号 + + + + + + + An estimate of the relative severity of the + vulnerability. The permitted values are shown below. There is no default value. + 脆弱性の相対的な深刻度の指標を、以下の属性値(デフォルト値＝なし)から選択する。 + + + + + + + + + + + Low severity. + 低 + + + + + Medium severity. + 中 + + + + + High severity. + 高 + + + + + + + + + + + + + + + This is the vulnerability information was known to + the public or not. + 脆弱性情報の公開状況を、以下の属性値(デフォルト値＝なし)から選択する。なお、配布のガイドライン属性restrictionとは、独立した属性である。 + + + + + Public information. + 公開済み + + + + + Not public information. + 未公開 + + + + + + + + + Each vulnerability in such a way that one can + understand the type of software problem that produced the + vulnerability. + 脆弱性のタイプを記載する。タイプとして、NIST NVD で使用している VulnerabilityType + を使用する。 + + + + + + + + + + + + + + + + + + + A vulnerability can enable either a "local" and/or + "remote" attack. + + + + + + The remote attack is possible. + + + + + + Need the account and logon operation. + + + + + + Both attacks are possible. + + + + + + + + + + This attribute indicates whether product is + vulnerable or not. There is no default value. + 影響を受ける製品毎の項目フィールドであり、下記に示す脆弱性の影響有無を記述するaffectedstatus + 属性を持っている。 + + + + + Vulnerable to the issue. + 影響あり + + + + + Not Vulnerable to the + issue. + 影響なし + + + + + Under investigation or a status can't be + fixed. + 不明 + + + + + Vulnerable to the issue and continue to + investigate. + 影響あり調査中 + + + + + Not Vulnerable to the issue and continue to + investigate. + 影響なし調査中 + + + + + + + + + This attribute is Comparison operators for a version + or build number. + + + + + + + + + + + + + + + + The type of impact in relatively broad categories. + The permitted values are shown below. + 想定される影響のタイプを記載する。タイプとして、IODEF で使用している Impacttype + 属性を使用する。 + + + + + Administrative privileges were attempted or + obtained. + + + + + + A denial of service was attempted or + completed. + + + + + + An action on a file was attempted or + completed. + + + + + + A reconnaissance probe was attempted or + completed. + + + + + + User privileges were attempted or + obtained. + + + + + + The activity did not have any (technical) + impact. + + + + + + The impact of the activity is unknown. + + + + + + Anything not in one of the above + categories. + + + + + + + + + + The type of solution in relatively broad categories. + There is no default value. + 回避施策のタイプを、以下の属性値(デフォルト値＝なし)から選択する。 + + + + + This solution eliminates the vulnerability. + 脆弱性そのものを除去する施策である。 + + + + + workaround solution (which has a direct + effect to resolve the issue). + 暫定施策(直接的な効果)である。 + + + + + migration solution (which has a indirect + effect to resolve the issue). + 緩和施策(間接的な効果)である。 + + + + + There is no solution. + 回避施策はない。 + + + + + Under investigation or a status can't be + fixed. + 不明(調査中など) + + + + + + + + + The type of exploit in relatively broad categories. + There is no default value. + + + + + + An exploit code exists. + すぐに悪用できるコードが存在する。 + + + + + POC exists. + 動作確認に利用できるコードが存在する。 + + + + + Worm, Virus or Trojan Hose + exists. + ワーム、ウイルス、トロイの木馬などのコードが存在する。 + + + + + Information for the exploit + exists. + 手順紹介レベルの情報が存在する。 + + + + + There are no exploits for this + issue. + 上記のいずれも存在しない。 + + + + + Currently we are not aware of any exploits + for this issue. + 不明 + + + + + + + + + The name of the database to which the reference is + being made. The permitted values are shown below. There is no default value. + 参照する情報源を以下の属性値(デフォルト値＝なし)から選択する。 + + + + + Bugtraq. (=Security + Focus.) + Bugtraq (=Security Focus) + + + + + Common Vulnerabilities and Exposures + (CVE). + Common Vulnerabilities and Exposures + (CVE) + + + + + CERT/CC Vulnerability Catalog. (=CERT + Advisory) + CERT/CC Vulnerability Catalog (=CERT + Advisory) + + + + + A product vendor. + 製品開発ベンダ + + + + + A local database. + + + + + + Comments by person. + + + + + + Except for the above. + 上記以外 + + + + + JVN. + JVN + + + + + JVN Status Tracking Notes. + JVN Status Tracking Notes + + + + + IPA Security Center + IPA セキュリティセンター 緊急対策情報 + + + + + + IPA セキュリティセンター + + + + + + JPCERT 緊急報告 + + + + + JPCERT Report. + JPCERT Report + + + + + @police topics + @police topics + + + + + CERT Advisory. + CERT Advisory + + + + + US-CERT Cyber Security + Alerts. + US-CERT Cyber Security Alerts + + + + + US-CERT Vulnerability + Note. + US-CERT Vulnerability Note + + + + + US-CERT Technical Cyber Security + Alert. + US-CERT Technical Cyber Security + Alert + + + + + National Vulnerability Database + (NVD). + National Vulnerability Database + (NVD) + + + + + CIAC Bulletins. + CIAC Bulletins + + + + + AUSCERT. + AUSCERT + + + + + NISCC Vulnerability + Advisory. + NISCC Vulnerability Advisory + + + + + Common Vulnerabilities and Exposures + (CVE). + Common Vulnerabilities and Exposures + (CVE) + + + + + Open Vulnerability and Assessment Language + (OVAL). + Open Vulnerability and Assessment Language + (OVAL) + + + + + Secunia Advisory. + Secunia Advisory + + + + + Security Focus. + Security Focus + + + + + ISS X-Force Database. + ISS X-Force Database + + + + + OPEN SOURCE VULNERABILITY DATABASE + (OSVDB). + OPEN SOURCE VULNERABILITY DATABASE + (OSVDB) + + + + + ISS Security Alerts and + Advisories. + ISS Security Alerts and + Advisories + + + + + + X-Force セキュリティアラート＆アドバイザリ + + + + + SecurityTracker. + SecurityTracker + + + + + SecuriTeam. + SecuriTeam + + + + + FrSIRT Advisories. + FrSIRT Advisories + + + + + The SANS Institute Diary. + The SANS Institute Diary + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd b/src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd new file mode 100644 index 00000000..aea7d0db --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd @@ -0,0 +1,287 @@ + + + + + + +
    +

    About the XML namespace

    + +
    +

    + This schema document describes the XML namespace, in a form + suitable for import by other schema documents. +

    +

    + See + http://www.w3.org/XML/1998/namespace.html and + + http://www.w3.org/TR/REC-xml for information + about this namespace. +

    +

    + Note that local names in this namespace are intended to be + defined only by the World Wide Web Consortium or its subgroups. + The names currently defined in this namespace are listed below. + They should not be used with conflicting semantics by any Working + Group, specification, or document instance. +

    +

    + See further below in this document for more information about how to refer to this schema document from your own + XSD schema documents and about the + namespace-versioning policy governing this schema document. +

    +
    +
    +     +     + + + + +
    + +

    lang (as an attribute name)

    +

    + denotes an attribute whose value + is a language code for the natural language of the content of + any element; its value is inherited. This name is reserved + by virtue of its definition in the XML specification.

    + +
    +
    +

    Notes

    +

    + Attempting to install the relevant ISO 2- and 3-letter + codes as the enumerated possible values is probably never + going to be a realistic possibility. +

    +

    + See BCP 47 at + http://www.rfc-editor.org/rfc/bcp/bcp47.txt + and the IANA language subtag registry at + + http://www.iana.org/assignments/language-subtag-registry + for further information. +

    +

    + The union allows for the 'un-declaration' of xml:lang with + the empty string. +

    +
    +     +     + + + + + + + + + +     + + + + +
    + +

    space (as an attribute name)

    +

    + denotes an attribute whose + value is a keyword indicating what whitespace processing + discipline is intended for the content of the element; its + value is inherited. This name is reserved by virtue of its + definition in the XML specification.

    + +
    +     +     + + + + + + +     + + + +
    + +

    base (as an attribute name)

    +

    + denotes an attribute whose value + provides a URI to be used as the base for interpreting any + relative URIs in the scope of the element on which it + appears; its value is inherited. This name is reserved + by virtue of its definition in the XML Base specification.

    + +

    + See http://www.w3.org/TR/xmlbase/ + for information about this attribute. +

    +
    +     +     +     + + + + +
    + +

    id (as an attribute name)

    +

    + denotes an attribute whose value + should be interpreted as if declared to be of type ID. + This name is reserved by virtue of its definition in the + xml:id specification.

    + +

    + See http://www.w3.org/TR/xml-id/ + for information about this attribute. +

    +
    +     +     +     + + + + + + + + + + +
    + +

    Father (in any context at all)

    + +
    +

    + denotes Jon Bosak, the chair of + the original XML Working Group. This name is reserved by + the following decision of the W3C XML Plenary and + XML Coordination groups: +

    +
    +

    + In appreciation for his vision, leadership and + dedication the W3C XML Plenary on this 10th day of + February, 2000, reserves for Jon Bosak in perpetuity + the XML name "xml:Father". +

    +
    +
    +
    +     +     + + + +
    +

    About this schema document

    + +
    +

    + This schema defines attributes and an attribute group suitable + for use by schemas wishing to allow xml:base, + xml:lang, xml:space or + xml:id attributes on elements they define. +

    +

    + To enable this, such a schema must import this schema for + the XML namespace, e.g. as follows: +

    + 
    +          <schema . . .>
+           . . .
+           <import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2001/xml.xsd"/>
+
    +

    + or +

    + 
    +           <import namespace="http://www.w3.org/XML/1998/namespace"
+                      schemaLocation="http://www.w3.org/2009/01/xml.xsd"/>
+
    +

    + Subsequently, qualified reference to any of the attributes or the + group defined below will have the desired effect, e.g. +

    + 
    +          <type . . .>
+           . . .
+           <attributeGroup ref="xml:specialAttrs"/>
+
    +

    + will define a type which will schema-validate an instance element + with any of those attributes. +

    +
    +
    +     +     + + + +
    +

    Versioning policy for this schema document

    +
    +

    + In keeping with the XML Schema WG's standard versioning + policy, this schema document will persist at + + http://www.w3.org/2009/01/xml.xsd. +

    +

    + At the date of issue it can also be found at + + http://www.w3.org/2001/xml.xsd. +

    +

    + The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XML + Schema itself, or with the XML namespace itself. In other words, + if the XML Schema or XML namespaces change, the version of this + document at + http://www.w3.org/2001/xml.xsd + + will change accordingly; the version at + + http://www.w3.org/2009/01/xml.xsd + + will not change. +

    +

    + Previous dated (and unchanging) versions of this schema + document are at: +

    + +
    +
    +     +     + +     + diff --git a/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj b/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj new file mode 100644 index 00000000..96ffa805 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj @@ -0,0 +1,15 @@ + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/TASKS.md b/src/StellaOps.Feedser.Source.Jvn/TASKS.md new file mode 100644 index 00000000..6aec6706 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Jvn/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|MyJVN client (JVNRSS+VULDEF) with windowing|BE-Conn-JVN|Source.Common|**DONE** – windowed overview/detail fetch with rate limit handling implemented.| +|Schema/XSD validation and DTO sanitizer|BE-Conn-JVN, QA|Source.Common|**DONE** – parser validates XML against schema before DTO persistence.| +|Canonical mapping (aliases, jp_flags, refs)|BE-Conn-JVN|Models|**DONE** – mapper populates aliases, jp_flags, references while skipping non-actionable affected entries.| +|SourceState and idempotent dedupe|BE-Conn-JVN|Storage.Mongo|**DONE** – cursor tracks pending docs/mappings with resume support.| +|Golden fixtures and determinism tests|QA|Source.Jvn|**DONE** – deterministic snapshot test in `JvnConnectorTests` now passes with offline fixtures.| +|Async-safe overview query building|BE-Conn-JVN|Source.Common|DONE – `MyJvnClient` now builds query strings synchronously without blocking calls.| +|Reference dedupe + deterministic ordering|BE-Conn-JVN|Models|DONE – mapper merges by URL, retains richer metadata, sorts deterministically.| +|Console logging remediation|BE-Conn-JVN|Observability|**DONE** – connector now uses structured `ILogger` debug entries instead of console writes.| +|Offline fixtures for connector tests|QA|Source.Jvn|**DONE** – tests rely solely on canned HTTP responses and local fixtures.| +|Update VULDEF schema for vendor attribute|BE-Conn-JVN, QA|Source.Jvn|TODO – align XSD/builder so we can remove temporary schema tolerance once upstream publishes new schema.| diff --git a/src/StellaOps.Feedser.Source.Kev/Class1.cs b/src/StellaOps.Feedser.Source.Kev/Class1.cs new file mode 100644 index 00000000..ceaed415 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Kev; + +public sealed class KevConnectorPlugin : IConnectorPlugin +{ + public string Name => "kev"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj b/src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Kisa/Class1.cs b/src/StellaOps.Feedser.Source.Kisa/Class1.cs new file mode 100644 index 00000000..7497ac39 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kisa/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Kisa; + +public sealed class KisaConnectorPlugin : IConnectorPlugin +{ + public string Name => "kisa"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj b/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json new file mode 100644 index 00000000..cefb4b2f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json @@ -0,0 +1,6 @@ +{ + "resultsPerPage": 1, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": "this-should-be-an-array" +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json new file mode 100644 index 00000000..ed90665d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json @@ -0,0 +1,69 @@ +{ + "resultsPerPage": 2, + "startIndex": 0, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1000", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T10:00:00Z", + "lastModified": "2024-02-02T10:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability one." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_a:1.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-1001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T11:00:00Z", + "lastModified": "2024-02-02T11:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability two." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:P/AC:L/PR:L/UI:R/S:U/C:L/I:L/A:L", + "baseScore": 5.1, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_b:2.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json new file mode 100644 index 00000000..530ecdf3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json @@ -0,0 +1,69 @@ +{ + "resultsPerPage": 2, + "startIndex": 2, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1002", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T12:00:00Z", + "lastModified": "2024-02-02T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability three." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:L/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:N", + "baseScore": 3.1, + "baseSeverity": "LOW" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_c:3.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-1003", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T13:00:00Z", + "lastModified": "2024-02-02T13:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability four." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:A/AC:L/PR:N/UI:N/S:U/C:M/I:L/A:L", + "baseScore": 7.4, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_d:4.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json new file mode 100644 index 00000000..42cf57dc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json @@ -0,0 +1,38 @@ +{ + "resultsPerPage": 2, + "startIndex": 4, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1004", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T14:00:00Z", + "lastModified": "2024-02-02T14:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability five." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:L/I:H/A:L", + "baseScore": 7.9, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_e:5.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json new file mode 100644 index 00000000..8571956f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json @@ -0,0 +1,85 @@ +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 2, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T10:00:00Z", + "lastModified": "2024-01-02T10:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability one." } + ], + "references": [ + { + "url": "https://vendor.example.com/advisories/0001", + "source": "Vendor", + "tags": ["Vendor Advisory"] + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-0002", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T11:00:00Z", + "lastModified": "2024-01-02T11:00:00Z", + "descriptions": [ + { "lang": "fr", "value": "Description française" }, + { "lang": "en", "value": "Example vulnerability two." } + ], + "references": [ + { + "url": "https://cisa.example.gov/alerts/0002", + "source": "CISA", + "tags": ["US Government Resource"] + } + ], + "metrics": { + "cvssMetricV30": [ + { + "cvssData": { + "vectorString": "CVSS:3.0/AV:L/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", + "baseScore": 4.6, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_two:2.0:*:*:*:*:*:*:*" }, + { "vulnerable": false, "criteria": "cpe:2.3:a:example:product_two:2.1:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json new file mode 100644 index 00000000..bf68d9b9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json @@ -0,0 +1,45 @@ +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0003", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T12:00:00Z", + "lastModified": "2024-01-02T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability three." } + ], + "references": [ + { + "url": "https://example.org/patches/0003", + "source": "Vendor", + "tags": ["Patch"] + } + ], + "metrics": { + "cvssMetricV2": [ + { + "cvssData": { + "vectorString": "AV:N/AC:M/Au:N/C:P/I:P/A:P", + "baseScore": 6.8, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_three:3.5:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json new file mode 100644 index 00000000..f7be7b3a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json @@ -0,0 +1,51 @@ +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T10:00:00Z", + "lastModified": "2024-01-03T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability one updated." } + ], + "references": [ + { + "url": "https://vendor.example.com/advisories/0001", + "source": "Vendor", + "tags": ["Vendor Advisory"] + }, + { + "url": "https://kb.example.com/articles/0001", + "source": "KnowledgeBase", + "tags": ["Third Party Advisory"] + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", + "baseScore": 8.8, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.0:*:*:*:*:*:*:*" }, + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.1:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs new file mode 100644 index 00000000..da86c9f2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Nvd; +using StellaOps.Feedser.Source.Nvd.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Testing; +using StellaOps.Feedser.Testing; +using System.Net; + +namespace StellaOps.Feedser.Source.Nvd.Tests; + +[Collection("mongo-fixture")] +public sealed class NvdConnectorHarnessTests : IAsyncLifetime +{ + private readonly ConnectorTestHarness _harness; + + public NvdConnectorHarnessTests(MongoIntegrationFixture fixture) + { + _harness = new ConnectorTestHarness(fixture, new DateTimeOffset(2024, 1, 2, 12, 0, 0, TimeSpan.Zero), NvdOptions.HttpClientName); + } + + [Fact] + public async Task FetchAsync_MultiPagePersistsStartIndexMetadata() + { + await _harness.ResetAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var timeProvider = _harness.TimeProvider; + var handler = _harness.Handler; + + var windowStart = timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + + var firstUri = BuildRequestUri(options, windowStart, windowEnd); + var secondUri = BuildRequestUri(options, windowStart, windowEnd, startIndex: 2); + var thirdUri = BuildRequestUri(options, windowStart, windowEnd, startIndex: 4); + + handler.AddJsonResponse(firstUri, ReadFixture("nvd-multipage-1.json")); + handler.AddJsonResponse(secondUri, ReadFixture("nvd-multipage-2.json")); + handler.AddJsonResponse(thirdUri, ReadFixture("nvd-multipage-3.json")); + + await _harness.EnsureServiceProviderAsync(services => + { + services.AddNvdConnector(opts => + { + opts.BaseEndpoint = options.BaseEndpoint; + opts.WindowSize = options.WindowSize; + opts.WindowOverlap = options.WindowOverlap; + opts.InitialBackfill = options.InitialBackfill; + }); + }); + + var provider = _harness.ServiceProvider; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + + var firstDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, firstUri.ToString(), CancellationToken.None); + Assert.NotNull(firstDocument); + Assert.Equal("0", firstDocument!.Metadata["startIndex"]); + + var secondDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, secondUri.ToString(), CancellationToken.None); + Assert.NotNull(secondDocument); + Assert.Equal("2", secondDocument!.Metadata["startIndex"]); + + var thirdDocument = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, thirdUri.ToString(), CancellationToken.None); + Assert.NotNull(thirdDocument); + Assert.Equal("4", thirdDocument!.Metadata["startIndex"]); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pending) + ? pending.AsBsonArray + : new BsonArray(); + Assert.Equal(3, pendingDocuments.Count); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => _harness.ResetAsync(); + + private static Uri BuildRequestUri(NvdOptions options, DateTimeOffset start, DateTimeOffset end, int startIndex = 0) + { + var builder = new UriBuilder(options.BaseEndpoint); + var parameters = new Dictionary + { + ["lastModifiedStartDate"] = start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["lastModifiedEndDate"] = end.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["resultsPerPage"] = "2000", + }; + + if (startIndex > 0) + { + parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); + } + + builder.Query = string.Join("&", parameters.Select(kvp => $"{WebUtility.UrlEncode(kvp.Key)}={WebUtility.UrlEncode(kvp.Value)}")); + return builder.Uri; + } + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Nvd", "Fixtures", filename); + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs new file mode 100644 index 00000000..73d8e707 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs @@ -0,0 +1,607 @@ +using System; +using System.Collections.Concurrent; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Nvd; +using StellaOps.Feedser.Source.Nvd.Configuration; +using StellaOps.Feedser.Source.Nvd.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.ChangeHistory; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Nvd.Tests; + +[Collection("mongo-fixture")] +public sealed class NvdConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private FakeTimeProvider _timeProvider; + private readonly DateTimeOffset _initialNow; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public NvdConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _initialNow = new DateTimeOffset(2024, 1, 2, 12, 0, 0, TimeSpan.Zero); + _timeProvider = new FakeTimeProvider(_initialNow); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_FlowProducesCanonicalAdvisories() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var window1Start = _timeProvider.GetUtcNow() - options.InitialBackfill; + var window1End = window1Start + options.WindowSize; + _handler.AddJsonResponse(BuildRequestUri(options, window1Start, window1End), ReadFixture("nvd-window-1.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0001"); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0002"); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursorDocument = state!.Cursor; + Assert.NotNull(cursorDocument); + var lastWindowEnd = cursorDocument.TryGetValue("windowEnd", out var endValue) ? ReadDateTime(endValue) : (DateTimeOffset?)null; + Assert.Equal(window1End.UtcDateTime, lastWindowEnd?.UtcDateTime); + + _timeProvider.Advance(TimeSpan.FromHours(1)); + var now = _timeProvider.GetUtcNow(); + var startCandidate = (lastWindowEnd ?? window1End) - options.WindowOverlap; + var backfillLimit = now - options.InitialBackfill; + var window2Start = startCandidate < backfillLimit ? backfillLimit : startCandidate; + var window2End = window2Start + options.WindowSize; + if (window2End > now) + { + window2End = now; + } + + _handler.AddJsonResponse(BuildRequestUri(options, window2Start, window2End), ReadFixture("nvd-window-2.json")); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(3, advisories.Count); + Assert.Contains(advisories, advisory => advisory.AdvisoryKey == "CVE-2024-0003"); + + var documentStore = provider.GetRequiredService(); + var finalState = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(finalState); + var pendingDocuments = finalState!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) + ? pendingDocs.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocuments); + } + + [Fact] + public async Task FetchAsync_MultiPageWindowFetchesAllPages() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd), ReadFixture("nvd-multipage-1.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 2), ReadFixture("nvd-multipage-2.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 4), ReadFixture("nvd-multipage-3.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) + ? pendingDocs.AsBsonArray.Select(v => Guid.Parse(v.AsString)).ToArray() + : Array.Empty(); + Assert.Equal(3, pendingDocuments.Length); + + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var advisoryKeys = advisories.Select(advisory => advisory.AdvisoryKey).OrderBy(k => k).ToArray(); + + Assert.Equal(new[] { "CVE-2024-1000", "CVE-2024-1001", "CVE-2024-1002", "CVE-2024-1003", "CVE-2024-1004" }, advisoryKeys); + } + + [Fact] + public async Task Observability_RecordsCountersForSuccessfulFlow() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + using var collector = new MetricCollector(NvdDiagnostics.MeterName); + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd), ReadFixture("nvd-multipage-1.json")); + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd, startIndex: 2), ReadFixture("nvd-multipage-2.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + Assert.Equal(3, collector.GetValue("nvd.fetch.attempts")); + Assert.Equal(3, collector.GetValue("nvd.fetch.documents")); + Assert.Equal(0, collector.GetValue("nvd.fetch.failures")); + Assert.Equal(0, collector.GetValue("nvd.fetch.unchanged")); + Assert.Equal(3, collector.GetValue("nvd.parse.success")); + Assert.Equal(0, collector.GetValue("nvd.parse.failures")); + Assert.Equal(0, collector.GetValue("nvd.parse.quarantine")); + Assert.Equal(5, collector.GetValue("nvd.map.success")); + } + + [Fact] + public async Task ChangeHistory_RecordsDifferencesForModifiedCve() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + _handler.AddJsonResponse(BuildRequestUri(options, windowStart, windowEnd), ReadFixture("nvd-window-1.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var historyStore = provider.GetRequiredService(); + var historyEntries = await historyStore.GetRecentAsync("nvd", "CVE-2024-0001", 5, CancellationToken.None); + Assert.Empty(historyEntries); + + _timeProvider.Advance(TimeSpan.FromHours(2)); + var now = _timeProvider.GetUtcNow(); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + + var cursorDocument = state!.Cursor; + var lastWindowEnd = cursorDocument.TryGetValue("windowEnd", out var endValue) ? ReadDateTime(endValue) : (DateTimeOffset?)null; + var startCandidate = (lastWindowEnd ?? windowEnd) - options.WindowOverlap; + var backfillLimit = now - options.InitialBackfill; + var window2Start = startCandidate < backfillLimit ? backfillLimit : startCandidate; + var window2End = window2Start + options.WindowSize; + if (window2End > now) + { + window2End = now; + } + + _handler.AddJsonResponse(BuildRequestUri(options, window2Start, window2End), ReadFixture("nvd-window-update.json")); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var updatedAdvisory = await advisoryStore.FindAsync("CVE-2024-0001", CancellationToken.None); + Assert.NotNull(updatedAdvisory); + Assert.Equal("high", updatedAdvisory!.Severity); + + historyEntries = await historyStore.GetRecentAsync("nvd", "CVE-2024-0001", 5, CancellationToken.None); + Assert.NotEmpty(historyEntries); + var latest = historyEntries[0]; + Assert.Equal("nvd", latest.SourceName); + Assert.Equal("CVE-2024-0001", latest.AdvisoryKey); + Assert.NotNull(latest.PreviousHash); + Assert.NotEqual(latest.PreviousHash, latest.CurrentHash); + Assert.Contains(latest.Changes, change => change.Field == "severity" && change.ChangeType == "Modified"); + Assert.Contains(latest.Changes, change => change.Field == "references" && change.ChangeType == "Modified"); + } + + [Fact] + public async Task ParseAsync_InvalidSchema_QuarantinesDocumentAndEmitsMetric() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + using var collector = new MetricCollector(NvdDiagnostics.MeterName); + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + var requestUri = BuildRequestUri(options, windowStart, windowEnd); + + _handler.AddJsonResponse(requestUri, ReadFixture("nvd-invalid-schema.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(NvdConnectorPlugin.SourceName, requestUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Failed, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocs = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) ? pendingDocsValue.AsBsonArray : new BsonArray(); + Assert.Empty(pendingDocs); + var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) ? pendingMappingsValue.AsBsonArray : new BsonArray(); + Assert.Empty(pendingMappings); + + Assert.Equal(1, collector.GetValue("nvd.fetch.documents")); + Assert.Equal(0, collector.GetValue("nvd.parse.success")); + Assert.Equal(1, collector.GetValue("nvd.parse.quarantine")); + Assert.Equal(0, collector.GetValue("nvd.map.success")); + } + + [Fact] + public async Task ResetDatabase_IsolatesRuns() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var start = _timeProvider.GetUtcNow() - options.InitialBackfill; + var end = start + options.WindowSize; + _handler.AddJsonResponse(BuildRequestUri(options, start, end), ReadFixture("nvd-window-1.json")); + + await EnsureServiceProviderAsync(options); + var provider = _serviceProvider!; + var connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var firstRunKeys = (await advisoryStore.GetRecentAsync(10, CancellationToken.None)) + .Select(advisory => advisory.AdvisoryKey) + .OrderBy(k => k) + .ToArray(); + Assert.Equal(new[] { "CVE-2024-0001", "CVE-2024-0002" }, firstRunKeys); + + await ResetDatabaseAsync(); + + options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + start = _timeProvider.GetUtcNow() - options.InitialBackfill; + end = start + options.WindowSize; + _handler.AddJsonResponse(BuildRequestUri(options, start, end), ReadFixture("nvd-window-2.json")); + + await EnsureServiceProviderAsync(options); + provider = _serviceProvider!; + connector = new NvdConnectorPlugin().Create(provider); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + advisoryStore = provider.GetRequiredService(); + var secondRunKeys = (await advisoryStore.GetRecentAsync(10, CancellationToken.None)) + .Select(advisory => advisory.AdvisoryKey) + .OrderBy(k => k) + .ToArray(); + Assert.Equal(new[] { "CVE-2024-0003" }, secondRunKeys); + } + + private async Task EnsureServiceProviderAsync(NvdOptions options) + { + if (_serviceProvider is not null) + { + return; + } + + _serviceProvider = await CreateServiceProviderAsync(options, _handler); + } + + [Fact] + public async Task Resume_CompletesPendingDocumentsAfterRestart() + { + await ResetDatabaseAsync(); + + var options = new NvdOptions + { + BaseEndpoint = new Uri("https://nvd.example.test/api"), + WindowSize = TimeSpan.FromHours(1), + WindowOverlap = TimeSpan.FromMinutes(5), + InitialBackfill = TimeSpan.FromHours(2), + }; + + var windowStart = _timeProvider.GetUtcNow() - options.InitialBackfill; + var windowEnd = windowStart + options.WindowSize; + var requestUri = BuildRequestUri(options, windowStart, windowEnd); + + var fetchHandler = new CannedHttpMessageHandler(); + fetchHandler.AddJsonResponse(requestUri, ReadFixture("nvd-window-1.json")); + + Guid[] pendingDocumentIds; + await using (var fetchProvider = await CreateServiceProviderAsync(options, fetchHandler)) + { + var connector = new NvdConnectorPlugin().Create(fetchProvider); + await connector.FetchAsync(fetchProvider, CancellationToken.None); + + var stateRepository = fetchProvider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pending = state!.Cursor.TryGetValue("pendingDocuments", out var value) + ? value.AsBsonArray + : new BsonArray(); + Assert.NotEmpty(pending); + pendingDocumentIds = pending.Select(v => Guid.Parse(v.AsString)).ToArray(); + } + + var resumeHandler = new CannedHttpMessageHandler(); + await using (var resumeProvider = await CreateServiceProviderAsync(options, resumeHandler)) + { + var resumeConnector = new NvdConnectorPlugin().Create(resumeProvider); + + await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None); + await resumeConnector.MapAsync(resumeProvider, CancellationToken.None); + + var documentStore = resumeProvider.GetRequiredService(); + foreach (var documentId in pendingDocumentIds) + { + var document = await documentStore.FindAsync(documentId, CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + } + + var advisoryStore = resumeProvider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.NotEmpty(advisories); + + var stateRepository = resumeProvider.GetRequiredService(); + var finalState = await stateRepository.TryGetAsync(NvdConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(finalState); + var cursor = finalState!.Cursor; + var finalPendingDocs = cursor.TryGetValue("pendingDocuments", out var pendingDocs) ? pendingDocs.AsBsonArray : new BsonArray(); + Assert.Empty(finalPendingDocs); + var finalPendingMappings = cursor.TryGetValue("pendingMappings", out var pendingMappings) ? pendingMappings.AsBsonArray : new BsonArray(); + Assert.Empty(finalPendingMappings); + } + } + + private Task ResetDatabaseAsync() + { + return ResetDatabaseInternalAsync(); + } + + private async Task CreateServiceProviderAsync(NvdOptions options, CannedHttpMessageHandler handler) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(storageOptions => + { + storageOptions.ConnectionString = _fixture.Runner.ConnectionString; + storageOptions.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + storageOptions.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddNvdConnector(configure: opts => + { + opts.BaseEndpoint = options.BaseEndpoint; + opts.WindowSize = options.WindowSize; + opts.WindowOverlap = options.WindowOverlap; + opts.InitialBackfill = options.InitialBackfill; + }); + + services.Configure(NvdOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private async Task ResetDatabaseInternalAsync() + { + if (_serviceProvider is not null) + { + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider.Dispose(); + } + + _serviceProvider = null; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + _timeProvider = new FakeTimeProvider(_initialNow); + } + + private sealed class MetricCollector : IDisposable + { + private readonly MeterListener _listener; + private readonly ConcurrentDictionary _measurements = new(StringComparer.OrdinalIgnoreCase); + + public MetricCollector(string meterName) + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == meterName) + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + + _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + _measurements.AddOrUpdate(instrument.Name, measurement, (_, existing) => existing + measurement); + }); + + _listener.Start(); + } + + public long GetValue(string instrumentName) + => _measurements.TryGetValue(instrumentName, out var value) ? value : 0; + + public void Dispose() + { + _listener.Dispose(); + } + } + + private static Uri BuildRequestUri(NvdOptions options, DateTimeOffset start, DateTimeOffset end, int startIndex = 0) + { + var builder = new UriBuilder(options.BaseEndpoint); + var parameters = new Dictionary + { + ["lastModifiedStartDate"] = start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["lastModifiedEndDate"] = end.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["resultsPerPage"] = "2000", + }; + + if (startIndex > 0) + { + parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); + } + + builder.Query = string.Join("&", parameters.Select(static kvp => $"{System.Net.WebUtility.UrlEncode(kvp.Key)}={System.Net.WebUtility.UrlEncode(kvp.Value)}")); + return builder.Uri; + } + + private static DateTimeOffset? ReadDateTime(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Source", "Nvd", "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var secondary = Path.Combine(baseDirectory, "Nvd", "Fixtures", filename); + if (File.Exists(secondary)) + { + return File.ReadAllText(secondary); + } + + throw new FileNotFoundException($"Fixture '{filename}' was not found in the test output directory."); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await ResetDatabaseInternalAsync(); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj b/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj new file mode 100644 index 00000000..a00e03f5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Nvd/AGENTS.md b/src/StellaOps.Feedser.Source.Nvd/AGENTS.md new file mode 100644 index 00000000..d6e6a5fc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/AGENTS.md @@ -0,0 +1,27 @@ +# AGENTS +## Role +Connector for NVD API v2: fetch, validate, map CVE items to canonical advisories, including CVSS/CWE/CPE as aliases/references. +## Scope +- Windowed fetch by modified range (6-12h default) with pagination; respect rate limits. +- Parse NVD JSON; validate against schema; extract CVSS v3/v4 metrics, CWE IDs, configurations.cpeMatch. +- Map to Advisory: primary id='CVE-YYYY-NNNN'; references; AffectedPackage entries for CPE (type=cpe) and optional vendor tags. +- Optional change-history capture: store previous payload hashes and diff summaries for auditing modified CVEs. +- Watermark: last successful modified_end; handle partial windows with overlap to avoid misses. +## Participants +- Merge engine reconciles NVD with PSIRT/OVAL (NVD yields to OVAL for OS packages). +- KEV connector may flag some CVEs; NVD severity is preserved but not overridden by KEV. +- Exporters consume canonical advisories. +## Interfaces & contracts +- Job kinds: nvd:fetch, nvd:parse, nvd:map. +- Input params: windowHours, since, until; safe defaults in FeedserOptions. +- Output: raw documents, sanitized DTOs, mapped advisories + provenance (document, parser). +## In/Out of scope +In: registry-level data, references, generic CPEs. +Out: authoritative distro package ranges; vendor patch states. +## Observability & security expectations +- Metrics: nvd.fetch.pages, items.count, schema.fail, map.advisories, window.advance; structured logs include window bounds and etag hits. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Nvd.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs b/src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs new file mode 100644 index 00000000..c8014da3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs @@ -0,0 +1,57 @@ +namespace StellaOps.Feedser.Source.Nvd.Configuration; + +public sealed class NvdOptions +{ + /// + /// Name of the HttpClient registered for NVD fetches. + /// + public const string HttpClientName = "nvd"; + + /// + /// Base API endpoint for CVE feed queries. + /// + public Uri BaseEndpoint { get; set; } = new("https://services.nvd.nist.gov/rest/json/cves/2.0"); + + /// + /// Duration of each modified window fetch. + /// + public TimeSpan WindowSize { get; set; } = TimeSpan.FromHours(4); + + /// + /// Overlap added when advancing the sliding window to cover upstream delays. + /// + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Maximum look-back period used when the connector first starts or state is empty. + /// + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(7); + + public void Validate() + { + if (BaseEndpoint is null) + { + throw new InvalidOperationException("NVD base endpoint must be configured."); + } + + if (!BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("NVD base endpoint must be an absolute URI."); + } + + if (WindowSize <= TimeSpan.Zero) + { + throw new InvalidOperationException("Window size must be positive."); + } + + if (WindowOverlap < TimeSpan.Zero || WindowOverlap >= WindowSize) + { + throw new InvalidOperationException("Window overlap must be non-negative and less than the window size."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Initial backfill duration must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs new file mode 100644 index 00000000..3f967e12 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs @@ -0,0 +1,64 @@ +using System.Linq; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common.Cursors; + +namespace StellaOps.Feedser.Source.Nvd.Internal; + +internal sealed record NvdCursor( + TimeWindowCursorState Window, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static NvdCursor Empty { get; } = new(TimeWindowCursorState.Empty, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + Window.WriteTo(document); + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); + return document; + } + + public static NvdCursor FromBsonDocument(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var window = TimeWindowCursorState.FromBsonDocument(document); + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new NvdCursor(window, pendingDocuments, pendingMappings); + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.AsString, out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + public NvdCursor WithWindow(TimeWindow window) + => this with { Window = Window.WithWindow(window) }; + + public NvdCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public NvdCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs new file mode 100644 index 00000000..5d7b40ec --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs @@ -0,0 +1,76 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Nvd.Internal; + +public sealed class NvdDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Nvd"; + public const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _parseQuarantine; + private readonly Counter _mapSuccess; + + public NvdDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter( + name: "nvd.fetch.attempts", + unit: "operations", + description: "Number of NVD fetch operations attempted, including paginated windows."); + _fetchDocuments = _meter.CreateCounter( + name: "nvd.fetch.documents", + unit: "documents", + description: "Count of NVD documents fetched and persisted."); + _fetchFailures = _meter.CreateCounter( + name: "nvd.fetch.failures", + unit: "operations", + description: "Count of NVD fetch attempts that resulted in an error or missing document."); + _fetchUnchanged = _meter.CreateCounter( + name: "nvd.fetch.unchanged", + unit: "operations", + description: "Count of NVD fetch attempts returning 304 Not Modified."); + _parseSuccess = _meter.CreateCounter( + name: "nvd.parse.success", + unit: "documents", + description: "Count of NVD documents successfully validated and converted into DTOs."); + _parseFailures = _meter.CreateCounter( + name: "nvd.parse.failures", + unit: "documents", + description: "Count of NVD documents that failed parsing due to missing content or read errors."); + _parseQuarantine = _meter.CreateCounter( + name: "nvd.parse.quarantine", + unit: "documents", + description: "Count of NVD documents quarantined due to schema validation failures."); + _mapSuccess = _meter.CreateCounter( + name: "nvd.map.success", + unit: "advisories", + description: "Count of canonical advisories produced by NVD mapping."); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void ParseQuarantine() => _parseQuarantine.Add(1); + + public void MapSuccess(long count = 1) => _mapSuccess.Add(count); + + public Meter Meter => _meter; + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs new file mode 100644 index 00000000..3f1c01d8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs @@ -0,0 +1,293 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Nvd.Internal; + +internal static class NvdMapper +{ + public static IReadOnlyList Map(JsonDocument document, DocumentRecord sourceDocument, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(sourceDocument); + + if (!document.RootElement.TryGetProperty("vulnerabilities", out var vulnerabilities) || vulnerabilities.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var advisories = new List(vulnerabilities.GetArrayLength()); + var index = 0; + foreach (var vulnerability in vulnerabilities.EnumerateArray()) + { + if (!vulnerability.TryGetProperty("cve", out var cve) || cve.ValueKind != JsonValueKind.Object) + { + index++; + continue; + } + + if (!cve.TryGetProperty("id", out var idElement) || idElement.ValueKind != JsonValueKind.String) + { + index++; + continue; + } + + var cveId = idElement.GetString(); + var advisoryKey = string.IsNullOrWhiteSpace(cveId) + ? $"nvd:{sourceDocument.Id:N}:{index}" + : cveId; + + var published = TryGetDateTime(cve, "published"); + var modified = TryGetDateTime(cve, "lastModified"); + var description = GetNormalizedDescription(cve); + + var references = GetReferences(cve, sourceDocument, recordedAt); + var affectedPackages = GetAffectedPackages(cve, sourceDocument, recordedAt); + var cvssMetrics = GetCvssMetrics(cve, sourceDocument, recordedAt, out var severity); + + var provenance = new[] + { + new AdvisoryProvenance(NvdConnectorPlugin.SourceName, "document", sourceDocument.Uri, sourceDocument.FetchedAt), + new AdvisoryProvenance(NvdConnectorPlugin.SourceName, "mapping", string.IsNullOrWhiteSpace(cveId) ? advisoryKey : cveId, recordedAt), + }; + + var title = string.IsNullOrWhiteSpace(cveId) ? advisoryKey : cveId; + + var aliasCandidates = new List(capacity: 2); + if (!string.IsNullOrWhiteSpace(cveId)) + { + aliasCandidates.Add(cveId); + } + + aliasCandidates.Add(advisoryKey); + + var advisory = new Advisory( + advisoryKey: advisoryKey, + title: title, + summary: string.IsNullOrEmpty(description.Text) ? null : description.Text, + language: description.Language, + published: published, + modified: modified, + severity: severity, + exploitKnown: false, + aliases: aliasCandidates, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: cvssMetrics, + provenance: provenance); + + advisories.Add(advisory); + index++; + } + + return advisories; + } + + private static NormalizedDescription GetNormalizedDescription(JsonElement cve) + { + var candidates = new List(); + + if (cve.TryGetProperty("descriptions", out var descriptions) && descriptions.ValueKind == JsonValueKind.Array) + { + foreach (var item in descriptions.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + continue; + } + + var text = item.TryGetProperty("value", out var valueElement) && valueElement.ValueKind == JsonValueKind.String + ? valueElement.GetString() + : null; + var lang = item.TryGetProperty("lang", out var langElement) && langElement.ValueKind == JsonValueKind.String + ? langElement.GetString() + : null; + + if (!string.IsNullOrWhiteSpace(text)) + { + candidates.Add(new LocalizedText(text, lang)); + } + } + } + + return DescriptionNormalizer.Normalize(candidates); + } + + private static DateTimeOffset? TryGetDateTime(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.String) + { + return null; + } + + return DateTimeOffset.TryParse(property.GetString(), out var parsed) ? parsed : null; + } + + private static IReadOnlyList GetReferences(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt) + { + var references = new List(); + if (!cve.TryGetProperty("references", out var referencesElement) || referencesElement.ValueKind != JsonValueKind.Array) + { + return references; + } + + foreach (var reference in referencesElement.EnumerateArray()) + { + if (!reference.TryGetProperty("url", out var urlElement) || urlElement.ValueKind != JsonValueKind.String) + { + continue; + } + + var url = urlElement.GetString(); + if (string.IsNullOrWhiteSpace(url) || !Validation.LooksLikeHttpUrl(url)) + { + continue; + } + + var sourceTag = reference.TryGetProperty("source", out var sourceElement) ? sourceElement.GetString() : null; + string? kind = null; + if (reference.TryGetProperty("tags", out var tagsElement) && tagsElement.ValueKind == JsonValueKind.Array) + { + kind = tagsElement.EnumerateArray().Select(static t => t.GetString()).FirstOrDefault(static tag => !string.IsNullOrWhiteSpace(tag))?.ToLowerInvariant(); + } + + references.Add(new AdvisoryReference( + url: url, + kind: kind, + sourceTag: sourceTag, + summary: null, + provenance: new AdvisoryProvenance(NvdConnectorPlugin.SourceName, "reference", document.Uri, recordedAt))); + } + + return references; + } + + private static IReadOnlyList GetAffectedPackages(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt) + { + var packages = new List(); + if (!cve.TryGetProperty("configurations", out var configurations) || configurations.ValueKind != JsonValueKind.Object) + { + return packages; + } + + if (!configurations.TryGetProperty("nodes", out var nodes) || nodes.ValueKind != JsonValueKind.Array) + { + return packages; + } + + foreach (var node in nodes.EnumerateArray()) + { + if (!node.TryGetProperty("cpeMatch", out var matches) || matches.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var match in matches.EnumerateArray()) + { + if (!match.TryGetProperty("criteria", out var criteriaElement) || criteriaElement.ValueKind != JsonValueKind.String) + { + continue; + } + + var criteria = criteriaElement.GetString(); + if (string.IsNullOrWhiteSpace(criteria)) + { + continue; + } + + if (!IdentifierNormalizer.TryNormalizeCpe(criteria, out var normalizedCpe)) + { + continue; + } + + var provenance = new AdvisoryProvenance(NvdConnectorPlugin.SourceName, "cpe", document.Uri, recordedAt); + packages.Add(new AffectedPackage( + type: AffectedPackageTypes.Cpe, + identifier: normalizedCpe!, + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: new[] { provenance })); + } + } + + return packages; + } + + private static IReadOnlyList GetCvssMetrics(JsonElement cve, DocumentRecord document, DateTimeOffset recordedAt, out string? severity) + { + severity = null; + if (!cve.TryGetProperty("metrics", out var metrics) || metrics.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var sources = new[] { "cvssMetricV31", "cvssMetricV30", "cvssMetricV2" }; + foreach (var source in sources) + { + if (!metrics.TryGetProperty(source, out var array) || array.ValueKind != JsonValueKind.Array) + { + continue; + } + + var list = new List(); + foreach (var item in array.EnumerateArray()) + { + if (!item.TryGetProperty("cvssData", out var data) || data.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (!data.TryGetProperty("vectorString", out var vectorElement) || vectorElement.ValueKind != JsonValueKind.String) + { + continue; + } + + if (!data.TryGetProperty("baseScore", out var scoreElement) || scoreElement.ValueKind != JsonValueKind.Number) + { + continue; + } + + if (!data.TryGetProperty("baseSeverity", out var severityElement) || severityElement.ValueKind != JsonValueKind.String) + { + continue; + } + + var vector = vectorElement.GetString() ?? string.Empty; + var baseScore = scoreElement.GetDouble(); + var baseSeverity = severityElement.GetString(); + var versionToken = source switch + { + "cvssMetricV30" => "3.0", + "cvssMetricV31" => "3.1", + _ => "2.0", + }; + + if (!CvssMetricNormalizer.TryNormalize(versionToken, vector, baseScore, baseSeverity, out var normalized)) + { + continue; + } + + severity ??= normalized.BaseSeverity; + + list.Add(normalized.ToModel(new AdvisoryProvenance( + NvdConnectorPlugin.SourceName, + "cvss", + document.Uri, + recordedAt))); + } + + if (list.Count > 0) + { + return list; + } + } + + return Array.Empty(); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs new file mode 100644 index 00000000..b326165b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs @@ -0,0 +1,25 @@ +using System.IO; +using System.Reflection; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Nvd.Internal; + +internal static class NvdSchemaProvider +{ + private static readonly Lazy Cached = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => Cached.Value; + + private static JsonSchema LoadSchema() + { + var assembly = typeof(NvdSchemaProvider).GetTypeInfo().Assembly; + const string resourceName = "StellaOps.Feedser.Source.Nvd.Schemas.nvd-vulnerability.schema.json"; + + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); + using var reader = new StreamReader(stream); + var schemaText = reader.ReadToEnd(); + return JsonSchema.FromText(schemaText); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs b/src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs new file mode 100644 index 00000000..51d8031e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs @@ -0,0 +1,565 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Common.Cursors; +using StellaOps.Feedser.Source.Nvd.Configuration; +using StellaOps.Feedser.Source.Nvd.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.ChangeHistory; +using StellaOps.Plugin; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Nvd; + +public sealed class NvdConnector : IFeedConnector +{ + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IChangeHistoryStore _changeHistoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IJsonSchemaValidator _schemaValidator; + private readonly NvdOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly NvdDiagnostics _diagnostics; + + private static readonly JsonSchema Schema = NvdSchemaProvider.Schema; + + public NvdConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IChangeHistoryStore changeHistoryStore, + ISourceStateRepository stateRepository, + IJsonSchemaValidator schemaValidator, + IOptions options, + NvdDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _changeHistoryStore = changeHistoryStore ?? throw new ArgumentNullException(nameof(changeHistoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => NvdConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + var windowOptions = new TimeWindowCursorOptions + { + WindowSize = _options.WindowSize, + Overlap = _options.WindowOverlap, + InitialBackfill = _options.InitialBackfill, + }; + + var window = TimeWindowCursorPlanner.GetNextWindow(now, cursor.Window, windowOptions); + var requestUri = BuildRequestUri(window); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["windowStart"] = window.Start.ToString("O"), + ["windowEnd"] = window.End.ToString("O"), + }; + metadata["startIndex"] = "0"; + + try + { + _diagnostics.FetchAttempt(); + + var result = await _fetchService.FetchAsync( + new SourceFetchRequest( + NvdOptions.HttpClientName, + SourceName, + requestUri) + { + Metadata = metadata + }, + cancellationToken).ConfigureAwait(false); + + if (result.IsNotModified) + { + _diagnostics.FetchUnchanged(); + _logger.LogDebug("NVD window {Start} - {End} returned 304", window.Start, window.End); + await UpdateCursorAsync(cursor.WithWindow(window), cancellationToken).ConfigureAwait(false); + return; + } + + if (!result.IsSuccess || result.Document is null) + { + _diagnostics.FetchFailure(); + return; + } + + _diagnostics.FetchDocument(); + + var pendingDocuments = new HashSet(cursor.PendingDocuments) + { + result.Document.Id + }; + + var additionalDocuments = await FetchAdditionalPagesAsync( + window, + metadata, + result.Document, + cancellationToken).ConfigureAwait(false); + + foreach (var documentId in additionalDocuments) + { + pendingDocuments.Add(documentId); + } + + var updated = cursor + .WithWindow(window) + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(cursor.PendingMappings); + + await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "NVD fetch failed for {Uri}", requestUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingFetch = cursor.PendingDocuments.ToList(); + var pendingMapping = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + _diagnostics.ParseFailure(); + remainingFetch.Remove(documentId); + pendingMapping.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Document {DocumentId} is missing GridFS content; skipping", documentId); + _diagnostics.ParseFailure(); + remainingFetch.Remove(documentId); + pendingMapping.Remove(documentId); + continue; + } + + var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + try + { + using var jsonDocument = JsonDocument.Parse(rawBytes); + try + { + _schemaValidator.Validate(jsonDocument, Schema, document.Uri); + } + catch (JsonSchemaValidationException ex) + { + _logger.LogWarning(ex, "NVD schema validation failed for document {DocumentId} ({Uri})", document.Id, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingFetch.Remove(documentId); + pendingMapping.Remove(documentId); + _diagnostics.ParseQuarantine(); + continue; + } + + var sanitized = JsonSerializer.Serialize(jsonDocument.RootElement); + var payload = BsonDocument.Parse(sanitized); + + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "nvd.cve.v2", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + _diagnostics.ParseSuccess(); + + remainingFetch.Remove(documentId); + if (!pendingMapping.Contains(documentId)) + { + pendingMapping.Add(documentId); + } + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to parse NVD JSON payload for document {DocumentId} ({Uri})", document.Id, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingFetch.Remove(documentId); + pendingMapping.Remove(documentId); + _diagnostics.ParseFailure(); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingFetch) + .WithPendingMappings(pendingMapping); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMapping = cursor.PendingMappings.ToList(); + var now = _timeProvider.GetUtcNow(); + + foreach (var documentId in cursor.PendingMappings) + { + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + pendingMapping.Remove(documentId); + continue; + } + + var json = dto.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + using var jsonDocument = JsonDocument.Parse(json); + var advisories = NvdMapper.Map(jsonDocument, document, now) + .GroupBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .Select(static group => group.First()) + .ToArray(); + + var mappedCount = 0L; + foreach (var advisory in advisories) + { + if (string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) + { + _logger.LogWarning("Skipping advisory with missing key for document {DocumentId} ({Uri})", document.Id, document.Uri); + continue; + } + + var previous = await _advisoryStore.FindAsync(advisory.AdvisoryKey, cancellationToken).ConfigureAwait(false); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + if (previous is not null) + { + await RecordChangeHistoryAsync(advisory, previous, document, now, cancellationToken).ConfigureAwait(false); + } + mappedCount++; + } + + if (mappedCount > 0) + { + _diagnostics.MapSuccess(mappedCount); + } + + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMapping.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMapping); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task> FetchAdditionalPagesAsync( + TimeWindow window, + IReadOnlyDictionary baseMetadata, + DocumentRecord firstDocument, + CancellationToken cancellationToken) + { + if (firstDocument.GridFsId is null) + { + return Array.Empty(); + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(firstDocument.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Unable to download NVD first page {DocumentId} to evaluate pagination", firstDocument.Id); + return Array.Empty(); + } + + try + { + using var jsonDocument = JsonDocument.Parse(rawBytes); + var root = jsonDocument.RootElement; + + if (!TryReadInt32(root, "totalResults", out var totalResults) || !TryReadInt32(root, "resultsPerPage", out var resultsPerPage)) + { + return Array.Empty(); + } + + if (resultsPerPage <= 0 || totalResults <= resultsPerPage) + { + return Array.Empty(); + } + + var fetchedDocuments = new List(); + + foreach (var startIndex in PaginationPlanner.EnumerateAdditionalPages(totalResults, resultsPerPage)) + { + var metadata = new Dictionary(StringComparer.Ordinal); + foreach (var kvp in baseMetadata) + { + metadata[kvp.Key] = kvp.Value; + } + metadata["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); + + var request = new SourceFetchRequest( + NvdOptions.HttpClientName, + SourceName, + BuildRequestUri(window, startIndex)) + { + Metadata = metadata + }; + + SourceFetchResult pageResult; + try + { + _diagnostics.FetchAttempt(); + pageResult = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "NVD fetch failed for page starting at {StartIndex}", startIndex); + throw; + } + + if (pageResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + continue; + } + + if (!pageResult.IsSuccess || pageResult.Document is null) + { + _diagnostics.FetchFailure(); + _logger.LogWarning("NVD fetch for page starting at {StartIndex} returned status {Status}", startIndex, pageResult.StatusCode); + continue; + } + + _diagnostics.FetchDocument(); + fetchedDocuments.Add(pageResult.Document.Id); + } + + return fetchedDocuments; + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to parse NVD first page {DocumentId} while determining pagination", firstDocument.Id); + return Array.Empty(); + } + } + + private static bool TryReadInt32(JsonElement root, string propertyName, out int value) + { + value = 0; + if (!root.TryGetProperty(propertyName, out var property) || property.ValueKind != JsonValueKind.Number) + { + return false; + } + + if (property.TryGetInt32(out var intValue)) + { + value = intValue; + return true; + } + + if (property.TryGetInt64(out var longValue)) + { + if (longValue > int.MaxValue) + { + value = int.MaxValue; + return true; + } + + value = (int)longValue; + return true; + } + + return false; + } + + private async Task RecordChangeHistoryAsync( + Advisory current, + Advisory previous, + DocumentRecord document, + DateTimeOffset capturedAt, + CancellationToken cancellationToken) + { + if (current.Equals(previous)) + { + return; + } + + var currentSnapshot = SnapshotSerializer.ToSnapshot(current); + var previousSnapshot = SnapshotSerializer.ToSnapshot(previous); + + if (string.Equals(currentSnapshot, previousSnapshot, StringComparison.Ordinal)) + { + return; + } + + var changes = ComputeChanges(previousSnapshot, currentSnapshot); + if (changes.Count == 0) + { + return; + } + + var documentHash = string.IsNullOrWhiteSpace(document.Sha256) + ? ComputeHash(currentSnapshot) + : document.Sha256; + + var record = new ChangeHistoryRecord( + Guid.NewGuid(), + SourceName, + current.AdvisoryKey, + document.Id, + documentHash, + ComputeHash(currentSnapshot), + ComputeHash(previousSnapshot), + currentSnapshot, + previousSnapshot, + changes, + capturedAt); + + await _changeHistoryStore.AddAsync(record, cancellationToken).ConfigureAwait(false); + } + + private static IReadOnlyList ComputeChanges(string previousSnapshot, string currentSnapshot) + { + using var previousDocument = JsonDocument.Parse(previousSnapshot); + using var currentDocument = JsonDocument.Parse(currentSnapshot); + + var previousRoot = previousDocument.RootElement; + var currentRoot = currentDocument.RootElement; + var fields = new HashSet(StringComparer.Ordinal); + + foreach (var property in previousRoot.EnumerateObject()) + { + fields.Add(property.Name); + } + + foreach (var property in currentRoot.EnumerateObject()) + { + fields.Add(property.Name); + } + + var changes = new List(); + foreach (var field in fields.OrderBy(static name => name, StringComparer.Ordinal)) + { + var hasPrevious = previousRoot.TryGetProperty(field, out var previousValue); + var hasCurrent = currentRoot.TryGetProperty(field, out var currentValue); + + if (!hasPrevious && hasCurrent) + { + changes.Add(new ChangeHistoryFieldChange(field, "Added", null, SerializeElement(currentValue))); + continue; + } + + if (hasPrevious && !hasCurrent) + { + changes.Add(new ChangeHistoryFieldChange(field, "Removed", SerializeElement(previousValue), null)); + continue; + } + + if (hasPrevious && hasCurrent && !JsonElement.DeepEquals(previousValue, currentValue)) + { + changes.Add(new ChangeHistoryFieldChange(field, "Modified", SerializeElement(previousValue), SerializeElement(currentValue))); + } + } + + return changes; + } + + private static string SerializeElement(JsonElement element) + => JsonSerializer.Serialize(element, new JsonSerializerOptions { WriteIndented = false }); + + private static string ComputeHash(string snapshot) + { + var bytes = Encoding.UTF8.GetBytes(snapshot); + var hash = SHA256.HashData(bytes); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return NvdCursor.FromBsonDocument(record?.Cursor); + } + + private async Task UpdateCursorAsync(NvdCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private Uri BuildRequestUri(TimeWindow window, int startIndex = 0) + { + var builder = new UriBuilder(_options.BaseEndpoint); + + var parameters = new Dictionary + { + ["lastModifiedStartDate"] = window.Start.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["lastModifiedEndDate"] = window.End.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"), + ["resultsPerPage"] = "2000", + }; + + if (startIndex > 0) + { + parameters["startIndex"] = startIndex.ToString(CultureInfo.InvariantCulture); + } + + builder.Query = string.Join("&", parameters.Select(static kvp => $"{System.Net.WebUtility.UrlEncode(kvp.Key)}={System.Net.WebUtility.UrlEncode(kvp.Value)}")); + return builder.Uri; + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs new file mode 100644 index 00000000..bc167313 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Nvd; + +public sealed class NvdConnectorPlugin : IConnectorPlugin +{ + public string Name => SourceName; + + public static string SourceName => "nvd"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs new file mode 100644 index 00000000..cd696700 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Nvd.Configuration; +using StellaOps.Feedser.Source.Nvd.Internal; + +namespace StellaOps.Feedser.Source.Nvd; + +public static class NvdServiceCollectionExtensions +{ + public static IServiceCollection AddNvdConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(NvdOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Nvd/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json b/src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json new file mode 100644 index 00000000..86170cbe --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["vulnerabilities"], + "properties": { + "resultsPerPage": { "type": "integer", "minimum": 0 }, + "startIndex": { "type": "integer", "minimum": 0 }, + "totalResults": { "type": "integer", "minimum": 0 }, + "vulnerabilities": { + "type": "array", + "items": { + "type": "object", + "required": ["cve"], + "properties": { + "cve": { + "type": "object", + "required": ["id", "published", "lastModified", "descriptions"], + "properties": { + "id": { "type": "string" }, + "published": { "type": "string", "format": "date-time" }, + "lastModified": { "type": "string", "format": "date-time" }, + "vulnStatus": { "type": "string" }, + "sourceIdentifier": { "type": "string" }, + "descriptions": { + "type": "array", + "items": { + "type": "object", + "required": ["lang", "value"], + "properties": { + "lang": { "type": "string" }, + "value": { "type": "string" } + } + } + }, + "references": { + "type": "array", + "items": { + "type": "object", + "required": ["url"], + "properties": { + "url": { "type": "string", "format": "uri" }, + "source": { "type": "string" }, + "tags": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "metrics": { + "type": "object", + "properties": { + "cvssMetricV2": { "$ref": "#/definitions/cvssMetricArray" }, + "cvssMetricV30": { "$ref": "#/definitions/cvssMetricArray" }, + "cvssMetricV31": { "$ref": "#/definitions/cvssMetricArray" } + } + }, + "configurations": { + "type": "object", + "properties": { + "nodes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "cpeMatch": { + "type": "array", + "items": { + "type": "object", + "properties": { + "vulnerable": { "type": "boolean" }, + "criteria": { "type": "string" } + }, + "required": ["criteria"], + "additionalProperties": true + } + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true, + "definitions": { + "cvssMetricArray": { + "type": "array", + "items": { + "type": "object", + "properties": { + "cvssData": { + "type": "object", + "required": ["vectorString", "baseScore", "baseSeverity"], + "properties": { + "vectorString": { "type": "string" }, + "baseScore": { "type": "number" }, + "baseSeverity": { "type": "string" } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj b/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj new file mode 100644 index 00000000..3673d8a6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Nvd/TASKS.md b/src/StellaOps.Feedser.Source.Nvd/TASKS.md new file mode 100644 index 00000000..d3a919e5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Nvd/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Fetch job with sliding modified windows|BE-Conn-Nvd|Source.Common|**DONE** – windowed fetch implemented with overlap and raw doc persistence.| +|DTO schema + validation|BE-Conn-Nvd|Source.Common|**DONE** – schema validator enforced before DTO persistence.| +|Mapper to canonical model|BE-Conn-Nvd|Models|**DONE** – `NvdMapper` populates CVSS/CWE/CPE data.| +|Watermark repo usage|BE-Conn-Nvd|Storage.Mongo|**DONE** – cursor tracks windowStart/windowEnd and updates SourceState.| +|Integration test fixture isolation|QA|Storage.Mongo|**DONE** – connector tests reset Mongo/time fixtures between runs to avoid cross-test bleed.| +|Tests: golden pages + resume|QA|Tests|**DONE** – snapshot and resume coverage added across `NvdConnectorTests`.| +|Observability|BE-Conn-Nvd|Core|**DONE** – `NvdDiagnostics` meter tracks attempts/documents/failures with collector tests.| +|Change history snapshotting|BE-Conn-Nvd|Storage.Mongo|DONE – connector now records per-CVE snapshots with top-level diff metadata whenever canonical advisories change.| +|Pagination for windows over page limit|BE-Conn-Nvd|Source.Common|**DONE** – additional page fetcher honors `startIndex`; covered by multipage tests.| +|Schema validation quarantine path|BE-Conn-Nvd|Storage.Mongo|**DONE** – schema failures mark documents failed and metrics assert quarantine.| diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs new file mode 100644 index 00000000..7f722145 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Osv; +using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Feedser.Source.Osv.Tests; + +public sealed class OsvMapperTests +{ + [Fact] + public void Map_NormalizesAliasesReferencesAndRanges() + { + var published = DateTimeOffset.UtcNow.AddDays(-2); + var modified = DateTimeOffset.UtcNow.AddDays(-1); + + using var databaseSpecificJson = JsonDocument.Parse("{}"); + using var ecosystemSpecificJson = JsonDocument.Parse("{}"); + + var dto = new OsvVulnerabilityDto + { + Id = "OSV-2025-TEST", + Summary = "Test summary", + Details = "Longer details for the advisory.", + Published = published, + Modified = modified, + Aliases = new[] { "CVE-2025-0001", "CVE-2025-0001", "GHSA-xxxx" }, + Related = new[] { "CVE-2025-0002" }, + References = new[] + { + new OsvReferenceDto { Url = "https://example.com/advisory", Type = "ADVISORY" }, + new OsvReferenceDto { Url = "https://example.com/advisory", Type = "ADVISORY" }, + new OsvReferenceDto { Url = "https://example.com/patch", Type = "PATCH" }, + }, + DatabaseSpecific = databaseSpecificJson.RootElement, + Severity = new[] + { + new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, + }, + Affected = new[] + { + new OsvAffectedPackageDto + { + Package = new OsvPackageDto + { + Ecosystem = "PyPI", + Name = "example", + Purl = "pkg:pypi/example", + }, + Ranges = new[] + { + new OsvRangeDto + { + Type = "SEMVER", + Events = new[] + { + new OsvEventDto { Introduced = "0" }, + new OsvEventDto { Fixed = "1.0.1" }, + } + } + }, + EcosystemSpecific = ecosystemSpecificJson.RootElement, + } + } + }; + + var document = new DocumentRecord( + Guid.NewGuid(), + OsvConnectorPlugin.SourceName, + "https://osv.dev/vulnerability/OSV-2025-TEST", + DateTimeOffset.UtcNow, + "sha256", + DocumentStatuses.PendingParse, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) + { + ["osv.ecosystem"] = "PyPI", + }, + null, + modified, + null, + null); + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + })); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, DateTimeOffset.UtcNow); + + var advisory = OsvMapper.Map(dto, document, dtoRecord, "PyPI"); + + Assert.Equal(dto.Id, advisory.AdvisoryKey); + Assert.Contains("CVE-2025-0002", advisory.Aliases); + Assert.Equal(4, advisory.Aliases.Length); + + Assert.Equal(2, advisory.References.Length); + Assert.Equal("https://example.com/advisory", advisory.References[0].Url); + Assert.Equal("https://example.com/patch", advisory.References[1].Url); + + Assert.Single(advisory.AffectedPackages); + var affected = advisory.AffectedPackages[0]; + Assert.Equal(AffectedPackageTypes.SemVer, affected.Type); + Assert.Single(affected.VersionRanges); + Assert.Equal("0", affected.VersionRanges[0].IntroducedVersion); + Assert.Equal("1.0.1", affected.VersionRanges[0].FixedVersion); + + Assert.Single(advisory.CvssMetrics); + Assert.Equal("3.1", advisory.CvssMetrics[0].Version); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj b/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj new file mode 100644 index 00000000..61506541 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Osv/AGENTS.md b/src/StellaOps.Feedser.Source.Osv/AGENTS.md new file mode 100644 index 00000000..0f0c8d7b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/AGENTS.md @@ -0,0 +1,27 @@ +# AGENTS +## Role +Connector for OSV.dev across ecosystems; authoritative SemVer/PURL ranges for OSS packages. +## Scope +- Fetch by ecosystem or time range; handle pagination and changed-since cursors. +- Parse OSV JSON; validate schema; capture introduced/fixed events, database_specific where relevant. +- Map to Advisory with AffectedPackage(type=semver, Identifier=PURL); preserve SemVer constraints and introduced/fixed chronology. +- Maintain per-ecosystem cursors and deduplicate runs via payload hashes to keep reruns idempotent. +## Participants +- Source.Common supplies HTTP clients, pagination helpers, and validators. +- Storage.Mongo persists documents, DTOs, advisories, and source_state cursors. +- Merge engine resolves OSV vs GHSA consistency; prefers SemVer data for libraries; distro OVAL still overrides OS packages. +- Exporters serialize per-ecosystem ranges untouched. +## Interfaces & contracts +- Job kinds: osv:fetch, osv:parse, osv:map (naming consistent with other connectors). +- Aliases include CVE/GHSA/OSV IDs; references include advisory/patch/release URLs. +- Provenance records method=parser and source=osv. +## In/Out of scope +In: SemVer+PURL accuracy for OSS ecosystems. +Out: vendor PSIRT and distro OVAL specifics. +## Observability & security expectations +- Metrics: osv.items, schema.fail, ranges.count, ecosystems.covered; logs include ecosystem and cursor values. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Osv.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs b/src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs new file mode 100644 index 00000000..64ce8b61 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs @@ -0,0 +1,81 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Osv.Configuration; + +public sealed class OsvOptions +{ + public const string HttpClientName = "source.osv"; + + public Uri BaseUri { get; set; } = new("https://osv-vulnerabilities.storage.googleapis.com/", UriKind.Absolute); + + public IReadOnlyList Ecosystems { get; set; } = new[] { "PyPI", "npm", "Maven", "Go", "crates" }; + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(14); + + public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromMinutes(10); + + public int MaxAdvisoriesPerFetch { get; set; } = 250; + + public string ArchiveFileName { get; set; } = "all.zip"; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(3); + + [MemberNotNull(nameof(BaseUri), nameof(Ecosystems), nameof(ArchiveFileName))] + public void Validate() + { + if (BaseUri is null || !BaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("OSV base URI must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(ArchiveFileName)) + { + throw new InvalidOperationException("OSV archive file name must be provided."); + } + + if (!ArchiveFileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("OSV archive file name must be a .zip resource."); + } + + if (Ecosystems is null || Ecosystems.Count == 0) + { + throw new InvalidOperationException("At least one OSV ecosystem must be configured."); + } + + foreach (var ecosystem in Ecosystems) + { + if (string.IsNullOrWhiteSpace(ecosystem)) + { + throw new InvalidOperationException("Ecosystem names cannot be null or whitespace."); + } + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Initial backfill window must be positive."); + } + + if (ModifiedTolerance < TimeSpan.Zero) + { + throw new InvalidOperationException("Modified tolerance cannot be negative."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException("Max advisories per fetch must be greater than zero."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("Request delay cannot be negative."); + } + + if (HttpTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("HTTP timeout must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs b/src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs new file mode 100644 index 00000000..8a3103c5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs @@ -0,0 +1,290 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Osv.Internal; + +internal sealed record OsvCursor( + IReadOnlyDictionary LastModifiedByEcosystem, + IReadOnlyDictionary> ProcessedIdsByEcosystem, + IReadOnlyDictionary ArchiveMetadataByEcosystem, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyDictionary EmptyLastModified = + new Dictionary(StringComparer.OrdinalIgnoreCase); + private static readonly IReadOnlyDictionary> EmptyProcessedIds = + new Dictionary>(StringComparer.OrdinalIgnoreCase); + private static readonly IReadOnlyDictionary EmptyArchiveMetadata = + new Dictionary(StringComparer.OrdinalIgnoreCase); + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyCollection EmptyStringList = Array.Empty(); + + public static OsvCursor Empty { get; } = new(EmptyLastModified, EmptyProcessedIds, EmptyArchiveMetadata, EmptyGuidList, EmptyGuidList); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastModifiedByEcosystem.Count > 0) + { + var lastModifiedDoc = new BsonDocument(); + foreach (var (ecosystem, timestamp) in LastModifiedByEcosystem) + { + lastModifiedDoc[ecosystem] = timestamp.HasValue ? BsonValue.Create(timestamp.Value.UtcDateTime) : BsonNull.Value; + } + + document["lastModified"] = lastModifiedDoc; + } + + if (ProcessedIdsByEcosystem.Count > 0) + { + var processedDoc = new BsonDocument(); + foreach (var (ecosystem, ids) in ProcessedIdsByEcosystem) + { + processedDoc[ecosystem] = new BsonArray(ids.Select(id => id)); + } + + document["processed"] = processedDoc; + } + + if (ArchiveMetadataByEcosystem.Count > 0) + { + var metadataDoc = new BsonDocument(); + foreach (var (ecosystem, metadata) in ArchiveMetadataByEcosystem) + { + var element = new BsonDocument(); + if (!string.IsNullOrWhiteSpace(metadata.ETag)) + { + element["etag"] = metadata.ETag; + } + + if (metadata.LastModified.HasValue) + { + element["lastModified"] = metadata.LastModified.Value.UtcDateTime; + } + + metadataDoc[ecosystem] = element; + } + + document["archive"] = metadataDoc; + } + + return document; + } + + public static OsvCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastModified = ReadLastModified(document.TryGetValue("lastModified", out var lastModifiedValue) ? lastModifiedValue : null); + var processed = ReadProcessedIds(document.TryGetValue("processed", out var processedValue) ? processedValue : null); + var archiveMetadata = ReadArchiveMetadata(document.TryGetValue("archive", out var archiveValue) ? archiveValue : null); + var pendingDocuments = ReadGuidList(document, "pendingDocuments"); + var pendingMappings = ReadGuidList(document, "pendingMappings"); + + return new OsvCursor(lastModified, processed, archiveMetadata, pendingDocuments, pendingMappings); + } + + public DateTimeOffset? GetLastModified(string ecosystem) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + return LastModifiedByEcosystem.TryGetValue(ecosystem, out var value) ? value : null; + } + + public bool HasProcessedId(string ecosystem, string id) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + ArgumentException.ThrowIfNullOrEmpty(id); + + return ProcessedIdsByEcosystem.TryGetValue(ecosystem, out var ids) + && ids.Contains(id, StringComparer.OrdinalIgnoreCase); + } + + public OsvCursor WithLastModified(string ecosystem, DateTimeOffset timestamp, IEnumerable processedIds) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + + var lastModified = new Dictionary(LastModifiedByEcosystem, StringComparer.OrdinalIgnoreCase) + { + [ecosystem] = timestamp.ToUniversalTime(), + }; + + var processed = new Dictionary>(ProcessedIdsByEcosystem, StringComparer.OrdinalIgnoreCase) + { + [ecosystem] = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyStringList, + }; + + return this with { LastModifiedByEcosystem = lastModified, ProcessedIdsByEcosystem = processed }; + } + + public OsvCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public OsvCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public OsvCursor AddProcessedId(string ecosystem, string id) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + ArgumentException.ThrowIfNullOrEmpty(id); + + var processed = new Dictionary>(ProcessedIdsByEcosystem, StringComparer.OrdinalIgnoreCase); + if (!processed.TryGetValue(ecosystem, out var ids)) + { + ids = EmptyStringList; + } + + var set = new HashSet(ids, StringComparer.OrdinalIgnoreCase) + { + id.Trim(), + }; + + processed[ecosystem] = set.ToArray(); + return this with { ProcessedIdsByEcosystem = processed }; + } + + public bool TryGetArchiveMetadata(string ecosystem, out OsvArchiveMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + return ArchiveMetadataByEcosystem.TryGetValue(ecosystem, out metadata!); + } + + public OsvCursor WithArchiveMetadata(string ecosystem, string? etag, DateTimeOffset? lastModified) + { + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + + var metadata = new Dictionary(ArchiveMetadataByEcosystem, StringComparer.OrdinalIgnoreCase) + { + [ecosystem] = new OsvArchiveMetadata(etag?.Trim(), lastModified?.ToUniversalTime()), + }; + + return this with { ArchiveMetadataByEcosystem = metadata }; + } + + private static IReadOnlyDictionary ReadLastModified(BsonValue? value) + { + if (value is not BsonDocument document) + { + return EmptyLastModified; + } + + var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in document.Elements) + { + if (element.Value is null || element.Value.IsBsonNull) + { + dictionary[element.Name] = null; + continue; + } + + dictionary[element.Name] = ParseDate(element.Value); + } + + return dictionary; + } + + private static IReadOnlyDictionary> ReadProcessedIds(BsonValue? value) + { + if (value is not BsonDocument document) + { + return EmptyProcessedIds; + } + + var dictionary = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var element in document.Elements) + { + if (element.Value is not BsonArray array) + { + continue; + } + + var ids = new List(array.Count); + foreach (var idValue in array) + { + if (idValue?.BsonType == BsonType.String) + { + var str = idValue.AsString.Trim(); + if (!string.IsNullOrWhiteSpace(str)) + { + ids.Add(str); + } + } + } + + dictionary[element.Name] = ids.Count == 0 + ? EmptyStringList + : ids.Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + } + + return dictionary; + } + + private static IReadOnlyDictionary ReadArchiveMetadata(BsonValue? value) + { + if (value is not BsonDocument document) + { + return EmptyArchiveMetadata; + } + + var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var element in document.Elements) + { + if (element.Value is not BsonDocument metadataDocument) + { + continue; + } + + string? etag = metadataDocument.TryGetValue("etag", out var etagValue) && etagValue.IsString ? etagValue.AsString : null; + DateTimeOffset? lastModified = metadataDocument.TryGetValue("lastModified", out var lastModifiedValue) + ? ParseDate(lastModifiedValue) + : null; + + dictionary[element.Name] = new OsvArchiveMetadata(etag, lastModified); + } + + return dictionary.Count == 0 ? EmptyArchiveMetadata : dictionary; + } + + private static IReadOnlyCollection ReadGuidList(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } +} + +internal sealed record OsvArchiveMetadata(string? ETag, DateTimeOffset? LastModified); diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs b/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs new file mode 100644 index 00000000..67ed4154 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs @@ -0,0 +1,391 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Feedser.Normalization.Text; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Source.Osv.Internal; + +internal static class OsvMapper +{ + private static readonly string[] SeverityOrder = { "none", "low", "medium", "high", "critical" }; + + public static Advisory Map( + OsvVulnerabilityDto dto, + DocumentRecord document, + DtoRecord dtoRecord, + string ecosystem) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + + var recordedAt = dtoRecord.ValidatedAt; + var fetchProvenance = new AdvisoryProvenance(OsvConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt); + var mappingProvenance = new AdvisoryProvenance(OsvConnectorPlugin.SourceName, "mapping", dto.Id, recordedAt); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var affectedPackages = BuildAffectedPackages(dto, ecosystem, recordedAt); + var cvssMetrics = BuildCvssMetrics(dto, recordedAt, out var severity); + + var normalizedDescription = DescriptionNormalizer.Normalize(new[] + { + new LocalizedText(dto.Details, "en"), + new LocalizedText(dto.Summary, "en"), + }); + + var title = string.IsNullOrWhiteSpace(dto.Summary) ? dto.Id : dto.Summary!.Trim(); + var summary = string.IsNullOrWhiteSpace(normalizedDescription.Text) ? dto.Summary : normalizedDescription.Text; + var language = string.IsNullOrWhiteSpace(normalizedDescription.Language) ? null : normalizedDescription.Language; + + return new Advisory( + dto.Id, + title, + summary, + language, + dto.Published?.ToUniversalTime(), + dto.Modified?.ToUniversalTime(), + severity, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics, + new[] { fetchProvenance, mappingProvenance }); + } + + private static IEnumerable BuildAliases(OsvVulnerabilityDto dto) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.Id, + }; + + if (dto.Aliases is not null) + { + foreach (var alias in dto.Aliases) + { + if (!string.IsNullOrWhiteSpace(alias)) + { + aliases.Add(alias.Trim()); + } + } + } + + if (dto.Related is not null) + { + foreach (var related in dto.Related) + { + if (!string.IsNullOrWhiteSpace(related)) + { + aliases.Add(related.Trim()); + } + } + } + + return aliases; + } + + private static IReadOnlyList BuildReferences(OsvVulnerabilityDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty(); + } + + var references = new List(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + var kind = NormalizeReferenceKind(reference.Type); + var provenance = new AdvisoryProvenance(OsvConnectorPlugin.SourceName, "reference", reference.Url, recordedAt); + + try + { + references.Add(new AdvisoryReference(reference.Url, kind, reference.Type, null, provenance)); + } + catch (ArgumentException) + { + // ignore invalid URLs + } + } + + if (references.Count <= 1) + { + return references; + } + + references.Sort(CompareReferences); + + var deduped = new List(references.Count); + string? lastUrl = null; + foreach (var reference in references) + { + if (lastUrl is not null && string.Equals(lastUrl, reference.Url, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + deduped.Add(reference); + lastUrl = reference.Url; + } + + return deduped; + } + + private static string? NormalizeReferenceKind(string? type) + { + if (string.IsNullOrWhiteSpace(type)) + { + return null; + } + + return type.Trim().ToLowerInvariant() switch + { + "advisory" => "advisory", + "exploit" => "exploit", + "fix" or "patch" => "patch", + "report" => "report", + "article" => "article", + _ => null, + }; + } + + private static IReadOnlyList BuildAffectedPackages(OsvVulnerabilityDto dto, string ecosystem, DateTimeOffset recordedAt) + { + if (dto.Affected is null || dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Affected.Count); + foreach (var affected in dto.Affected) + { + if (affected.Package is null) + { + continue; + } + + var identifier = DetermineIdentifier(affected.Package, ecosystem); + if (identifier is null) + { + continue; + } + + var provenance = new[] + { + new AdvisoryProvenance(OsvConnectorPlugin.SourceName, "affected", identifier, recordedAt), + }; + + var ranges = BuildVersionRanges(affected, recordedAt, identifier); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.SemVer, + identifier, + platform: affected.Package.Ecosystem, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: provenance)); + } + + return packages; + } + + private static IReadOnlyList BuildVersionRanges(OsvAffectedPackageDto affected, DateTimeOffset recordedAt, string identifier) + { + if (affected.Ranges is null || affected.Ranges.Count == 0) + { + return Array.Empty(); + } + + var ranges = new List(); + foreach (var range in affected.Ranges) + { + if (!"semver".Equals(range.Type, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var provenance = new AdvisoryProvenance(OsvConnectorPlugin.SourceName, "range", identifier, recordedAt); + if (range.Events is null || range.Events.Count == 0) + { + continue; + } + + string? introduced = null; + string? lastAffected = null; + + foreach (var evt in range.Events) + { + if (!string.IsNullOrWhiteSpace(evt.Introduced)) + { + introduced = evt.Introduced.Trim(); + lastAffected = null; + } + + if (!string.IsNullOrWhiteSpace(evt.LastAffected)) + { + lastAffected = evt.LastAffected.Trim(); + } + + if (!string.IsNullOrWhiteSpace(evt.Fixed)) + { + ranges.Add(new AffectedVersionRange( + "semver", + introduced, + evt.Fixed.Trim(), + lastAffected, + rangeExpression: null, + provenance: provenance)); + introduced = null; + lastAffected = null; + } + + if (!string.IsNullOrWhiteSpace(evt.Limit)) + { + lastAffected = evt.Limit.Trim(); + } + } + + if (introduced is not null || lastAffected is not null) + { + ranges.Add(new AffectedVersionRange( + "semver", + introduced, + fixedVersion: null, + lastAffected, + rangeExpression: null, + provenance: provenance)); + } + } + + return ranges.Count == 0 + ? Array.Empty() + : ranges; + } + + private static string? DetermineIdentifier(OsvPackageDto package, string ecosystem) + { + if (!string.IsNullOrWhiteSpace(package.Purl) + && IdentifierNormalizer.TryNormalizePackageUrl(package.Purl, out var normalized)) + { + return normalized; + } + + if (!string.IsNullOrWhiteSpace(package.Name)) + { + var name = package.Name.Trim(); + return string.IsNullOrWhiteSpace(package.Ecosystem) + ? $"{ecosystem}:{name}" + : $"{package.Ecosystem.Trim()}:{name}"; + } + + return null; + } + + private static IReadOnlyList BuildCvssMetrics(OsvVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity) + { + severity = null; + if (dto.Severity is null || dto.Severity.Count == 0) + { + return Array.Empty(); + } + + var metrics = new List(dto.Severity.Count); + var bestRank = -1; + + foreach (var severityEntry in dto.Severity) + { + if (string.IsNullOrWhiteSpace(severityEntry.Score)) + { + continue; + } + + if (!CvssMetricNormalizer.TryNormalize(severityEntry.Type, severityEntry.Score, null, null, out var normalized)) + { + continue; + } + + var provenance = new AdvisoryProvenance(OsvConnectorPlugin.SourceName, "cvss", severityEntry.Type ?? "osv", recordedAt); + metrics.Add(normalized.ToModel(provenance)); + + var rank = Array.IndexOf(SeverityOrder, normalized.BaseSeverity); + if (rank > bestRank) + { + bestRank = rank; + severity = normalized.BaseSeverity; + } + } + + return metrics; + } + + private static int CompareReferences(AdvisoryReference? left, AdvisoryReference? right) + { + if (ReferenceEquals(left, right)) + { + return 0; + } + + if (left is null) + { + return 1; + } + + if (right is null) + { + return -1; + } + + var compare = StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.Kind, right.Kind); + if (compare != 0) + { + return compare; + } + + compare = CompareNullable(left.SourceTag, right.SourceTag); + if (compare != 0) + { + return compare; + } + + return left.Provenance.RecordedAt.CompareTo(right.Provenance.RecordedAt); + } + + private static int CompareNullable(string? left, string? right) + { + if (left is null && right is null) + { + return 0; + } + + if (left is null) + { + return 1; + } + + if (right is null) + { + return -1; + } + + return StringComparer.Ordinal.Compare(left, right); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs b/src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs new file mode 100644 index 00000000..f7389454 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Osv.Internal; + +internal sealed record OsvVulnerabilityDto +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("details")] + public string? Details { get; init; } + + [JsonPropertyName("aliases")] + public IReadOnlyList? Aliases { get; init; } + + [JsonPropertyName("related")] + public IReadOnlyList? Related { get; init; } + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("severity")] + public IReadOnlyList? Severity { get; init; } + + [JsonPropertyName("references")] + public IReadOnlyList? References { get; init; } + + [JsonPropertyName("affected")] + public IReadOnlyList? Affected { get; init; } + + [JsonPropertyName("database_specific")] + public JsonElement DatabaseSpecific { get; init; } +} + +internal sealed record OsvSeverityDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("score")] + public string? Score { get; init; } +} + +internal sealed record OsvReferenceDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("url")] + public string? Url { get; init; } +} + +internal sealed record OsvAffectedPackageDto +{ + [JsonPropertyName("package")] + public OsvPackageDto? Package { get; init; } + + [JsonPropertyName("ranges")] + public IReadOnlyList? Ranges { get; init; } + + [JsonPropertyName("versions")] + public IReadOnlyList? Versions { get; init; } + + [JsonPropertyName("ecosystem_specific")] + public JsonElement EcosystemSpecific { get; init; } +} + +internal sealed record OsvPackageDto +{ + [JsonPropertyName("ecosystem")] + public string? Ecosystem { get; init; } + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("purl")] + public string? Purl { get; init; } +} + +internal sealed record OsvRangeDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("events")] + public IReadOnlyList? Events { get; init; } + + [JsonPropertyName("repo")] + public string? Repository { get; init; } +} + +internal sealed record OsvEventDto +{ + [JsonPropertyName("introduced")] + public string? Introduced { get; init; } + + [JsonPropertyName("fixed")] + public string? Fixed { get; init; } + + [JsonPropertyName("last_affected")] + public string? LastAffected { get; init; } + + [JsonPropertyName("limit")] + public string? Limit { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Osv/Jobs.cs b/src/StellaOps.Feedser.Source.Osv/Jobs.cs new file mode 100644 index 00000000..7fb08372 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Osv; + +internal static class OsvJobKinds +{ + public const string Fetch = "source:osv:fetch"; + public const string Parse = "source:osv:parse"; + public const string Map = "source:osv:map"; +} + +internal sealed class OsvFetchJob : IJob +{ + private readonly OsvConnector _connector; + + public OsvFetchJob(OsvConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class OsvParseJob : IJob +{ + private readonly OsvConnector _connector; + + public OsvParseJob(OsvConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class OsvMapJob : IJob +{ + private readonly OsvConnector _connector; + + public OsvMapJob(OsvConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs b/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs new file mode 100644 index 00000000..d1276991 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs @@ -0,0 +1,497 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Osv.Configuration; +using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Osv; + +public sealed class OsvConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNameCaseInsensitive = true, + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly OsvOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public OsvConnector( + IHttpClientFactory httpClientFactory, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => OsvConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var cursorState = cursor; + var remainingCapacity = _options.MaxAdvisoriesPerFetch; + + foreach (var ecosystem in _options.Ecosystems) + { + if (remainingCapacity <= 0) + { + break; + } + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var result = await FetchEcosystemAsync( + ecosystem, + cursorState, + pendingDocuments, + now, + remainingCapacity, + cancellationToken).ConfigureAwait(false); + + cursorState = result.Cursor; + remainingCapacity -= result.NewDocuments; + } + catch (Exception ex) + { + _logger.LogError(ex, "OSV fetch failed for ecosystem {Ecosystem}", ecosystem); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + cursorState = cursorState + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(cursor.PendingMappings); + + await UpdateCursorAsync(cursorState, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("OSV document {DocumentId} missing GridFS content", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to download OSV raw document {DocumentId}", document.Id); + throw; + } + + OsvVulnerabilityDto? dto; + try + { + dto = JsonSerializer.Deserialize(bytes, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to deserialize OSV document {DocumentId} ({Uri})", document.Id, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + if (dto is null || string.IsNullOrWhiteSpace(dto.Id)) + { + _logger.LogWarning("OSV document {DocumentId} produced empty payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var sanitized = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = MongoDB.Bson.BsonDocument.Parse(sanitized); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "osv.v1", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dto is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var payloadJson = dto.Payload.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + }); + + OsvVulnerabilityDto? osvDto; + try + { + osvDto = JsonSerializer.Deserialize(payloadJson, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize OSV DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (osvDto is null || string.IsNullOrWhiteSpace(osvDto.Id)) + { + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var ecosystem = document.Metadata is not null && document.Metadata.TryGetValue("osv.ecosystem", out var ecosystemValue) + ? ecosystemValue + : "unknown"; + + var advisory = OsvMapper.Map(osvDto, document, dto, ecosystem); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? OsvCursor.Empty : OsvCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(OsvCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private async Task<(OsvCursor Cursor, int NewDocuments)> FetchEcosystemAsync( + string ecosystem, + OsvCursor cursor, + HashSet pendingDocuments, + DateTimeOffset now, + int remainingCapacity, + CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(OsvOptions.HttpClientName); + client.Timeout = _options.HttpTimeout; + + var archiveUri = BuildArchiveUri(ecosystem); + using var request = new HttpRequestMessage(HttpMethod.Get, archiveUri); + + if (cursor.TryGetArchiveMetadata(ecosystem, out var archiveMetadata)) + { + if (!string.IsNullOrWhiteSpace(archiveMetadata.ETag)) + { + request.Headers.TryAddWithoutValidation("If-None-Match", archiveMetadata.ETag); + } + + if (archiveMetadata.LastModified.HasValue) + { + request.Headers.IfModifiedSince = archiveMetadata.LastModified.Value; + } + } + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + return (cursor, 0); + } + + response.EnsureSuccessStatusCode(); + + await using var archiveStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false); + + var existingLastModified = cursor.GetLastModified(ecosystem); + var processedIdsSet = cursor.ProcessedIdsByEcosystem.TryGetValue(ecosystem, out var processedIds) + ? new HashSet(processedIds, StringComparer.OrdinalIgnoreCase) + : new HashSet(StringComparer.OrdinalIgnoreCase); + + var currentMaxModified = existingLastModified ?? DateTimeOffset.MinValue; + var currentProcessedIds = new HashSet(processedIdsSet, StringComparer.OrdinalIgnoreCase); + var processedUpdated = false; + var newDocuments = 0; + + var minimumModified = existingLastModified.HasValue + ? existingLastModified.Value - _options.ModifiedTolerance + : now - _options.InitialBackfill; + + foreach (var entry in archive.Entries) + { + if (remainingCapacity <= 0) + { + break; + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (!entry.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + await using var entryStream = entry.Open(); + using var memory = new MemoryStream(); + await entryStream.CopyToAsync(memory, cancellationToken).ConfigureAwait(false); + var bytes = memory.ToArray(); + + OsvVulnerabilityDto? dto; + try + { + dto = JsonSerializer.Deserialize(bytes, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse OSV entry {Entry} for ecosystem {Ecosystem}", entry.FullName, ecosystem); + continue; + } + + if (dto is null || string.IsNullOrWhiteSpace(dto.Id)) + { + continue; + } + + var modified = (dto.Modified ?? dto.Published ?? DateTimeOffset.MinValue).ToUniversalTime(); + if (modified < minimumModified) + { + continue; + } + + if (existingLastModified.HasValue && modified < existingLastModified.Value - _options.ModifiedTolerance) + { + continue; + } + + if (modified < currentMaxModified - _options.ModifiedTolerance) + { + continue; + } + + if (modified == currentMaxModified && currentProcessedIds.Contains(dto.Id)) + { + continue; + } + + var documentUri = BuildDocumentUri(ecosystem, dto.Id); + var sha256 = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); + if (existing is not null && string.Equals(existing.Sha256, sha256, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, bytes, "application/json", null, cancellationToken).ConfigureAwait(false); + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["osv.ecosystem"] = ecosystem, + ["osv.id"] = dto.Id, + ["osv.modified"] = modified.ToString("O"), + }; + + var recordId = existing?.Id ?? Guid.NewGuid(); + var record = new DocumentRecord( + recordId, + SourceName, + documentUri, + _timeProvider.GetUtcNow(), + sha256, + DocumentStatuses.PendingParse, + "application/json", + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: modified, + GridFsId: gridFsId, + ExpiresAt: null); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + pendingDocuments.Add(upserted.Id); + newDocuments++; + remainingCapacity--; + + if (modified > currentMaxModified) + { + currentMaxModified = modified; + currentProcessedIds = new HashSet(StringComparer.OrdinalIgnoreCase) { dto.Id }; + processedUpdated = true; + } + else if (modified == currentMaxModified) + { + currentProcessedIds.Add(dto.Id); + processedUpdated = true; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + if (processedUpdated && currentMaxModified != DateTimeOffset.MinValue) + { + cursor = cursor.WithLastModified(ecosystem, currentMaxModified, currentProcessedIds); + } + else if (processedUpdated && existingLastModified.HasValue) + { + cursor = cursor.WithLastModified(ecosystem, existingLastModified.Value, currentProcessedIds); + } + + var etag = response.Headers.ETag?.Tag; + var lastModifiedHeader = response.Content.Headers.LastModified; + cursor = cursor.WithArchiveMetadata(ecosystem, etag, lastModifiedHeader); + + return (cursor, newDocuments); + } + + private Uri BuildArchiveUri(string ecosystem) + { + var trimmed = ecosystem.Trim('/'); + var baseUri = _options.BaseUri; + var builder = new UriBuilder(baseUri); + var path = builder.Path; + if (!path.EndsWith('/')) + { + path += "/"; + } + + path += $"{trimmed}/{_options.ArchiveFileName}"; + builder.Path = path; + return builder.Uri; + } + + private static string BuildDocumentUri(string ecosystem, string vulnerabilityId) + { + var safeId = vulnerabilityId.Replace(' ', '-'); + return $"https://osv-vulnerabilities.storage.googleapis.com/{ecosystem}/{safeId}.json"; + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs new file mode 100644 index 00000000..f995e3d6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Osv; + +public sealed class OsvConnectorPlugin : IConnectorPlugin +{ + public string Name => SourceName; + + public static string SourceName => "osv"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs new file mode 100644 index 00000000..60571863 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Osv.Configuration; + +namespace StellaOps.Feedser.Source.Osv; + +public sealed class OsvDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:osv"; + private const string FetchCron = "0,20,40 * * * *"; + private const string ParseCron = "5,25,45 * * * *"; + private const string MapCron = "10,30,50 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(15); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(20); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(20); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(10); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddOsvConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob( + OsvJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob( + OsvJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob( + OsvJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs new file mode 100644 index 00000000..20775191 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Osv.Configuration; + +namespace StellaOps.Feedser.Source.Osv; + +public static class OsvServiceCollectionExtensions +{ + public static IServiceCollection AddOsvConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(OsvOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseUri; + clientOptions.Timeout = options.HttpTimeout; + clientOptions.UserAgent = "StellaOps.Feedser.OSV/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseUri.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/zip"; + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj b/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj new file mode 100644 index 00000000..664b10ec --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj @@ -0,0 +1,20 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + <_Parameter1>StellaOps.Feedser.Tests + + + diff --git a/src/StellaOps.Feedser.Source.Osv/TASKS.md b/src/StellaOps.Feedser.Source.Osv/TASKS.md new file mode 100644 index 00000000..789efc3e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Osv/TASKS.md @@ -0,0 +1,13 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Ecosystem fetchers (npm, pypi, maven, go, crates)|BE-Conn-OSV|Source.Common|**DONE** – archive fetch loop iterates ecosystems with pagination + change gating.| +|OSV options & HttpClient configuration|BE-Conn-OSV|Source.Common|**DONE** – `OsvOptions` + `AddOsvConnector` configure allowlisted HttpClient.| +|DTO validation + sanitizer|BE-Conn-OSV|Source.Common|**DONE** – JSON deserialization sanitizes payloads before persistence; schema enforcement deferred.| +|Mapper to canonical SemVer ranges|BE-Conn-OSV|Models|**DONE** – `OsvMapper` emits SemVer ranges with provenance metadata.| +|Alias consolidation (GHSA/CVE)|BE-Merge|Merge|Identity graph enrichment for merge step.| +|Tests: snapshot per ecosystem|QA|Tests|Golden OSV records -> canonical advisories; deterministic ordering.| +|Cursor persistence and hash gating|BE-Conn-OSV|Storage.Mongo|**DONE** – `OsvCursor` tracks per-ecosystem metadata and SHA gating.| +|Parity checks vs GHSA data|QA|Merge|TODO – reconcile overlapping records for diff detection.| +|Connector DI routine & job registration|BE-Conn-OSV|Core|**DONE** – DI routine registers fetch/parse/map jobs with scheduler.| +|Implement OSV fetch/parse/map skeleton|BE-Conn-OSV|Source.Common|**DONE** – connector now persists documents, DTOs, and canonical advisories.| diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs b/src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs new file mode 100644 index 00000000..8afbacf2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Bdu/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ru.Bdu; + +public sealed class RuBduConnectorPlugin : IConnectorPlugin +{ + public string Name => "ru-bdu"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj b/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Class1.cs b/src/StellaOps.Feedser.Source.Ru.Nkcki/Class1.cs new file mode 100644 index 00000000..12fd0bd8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Ru.Nkcki; + +public sealed class RuNkckiConnectorPlugin : IConnectorPlugin +{ + public string Name => "ru-nkcki"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj b/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs new file mode 100644 index 00000000..d9b8f958 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs @@ -0,0 +1,351 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Vndr.Adobe; +using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Tests; + +[Collection("mongo-fixture")] +public sealed class AdobeConnectorFetchTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + + public AdobeConnectorFetchTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 9, 10, 0, 0, 0, TimeSpan.Zero)); + } + + [Fact] + public async Task Fetch_WindowsIndexAndPersistsCursor() + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler); + SeedIndex(handler); + SeedDetail(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = state!.Cursor; + var pendingDocuments = ExtractGuidList(cursor, "pendingDocuments"); + Assert.Equal(2, pendingDocuments.Count); + + // Re-seed responses to simulate unchanged fetch + SeedIndex(handler); + SeedDetail(handler); + await connector.FetchAsync(provider, CancellationToken.None); + + state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + cursor = state!.Cursor; + var afterPending = ExtractGuidList(cursor, "pendingDocuments"); + Assert.Equal(pendingDocuments.OrderBy(static id => id), afterPending.OrderBy(static id => id)); + + var fetchCache = cursor.TryGetValue("fetchCache", out var fetchCacheValue) && fetchCacheValue is BsonDocument cacheDoc + ? cacheDoc.Elements.Select(static e => e.Name).ToArray() + : Array.Empty(); + Assert.Contains("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", fetchCache); + Assert.Contains("https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html", fetchCache); + } + + [Fact] + public async Task Parse_ProducesDtoAndClearsPendingDocuments() + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler); + SeedIndex(handler); + SeedDetail(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var dtoStore = provider.GetRequiredService(); + var advisoryStore = provider.GetRequiredService(); + var psirtStore = provider.GetRequiredService(); + var stateRepository = provider.GetRequiredService(); + + var document = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", + CancellationToken.None); + + Assert.NotNull(document); + + var dtoRecord = await dtoStore.FindByDocumentIdAsync(document!.Id, CancellationToken.None); + Assert.NotNull(dtoRecord); + Assert.Equal("adobe.bulletin.v1", dtoRecord!.SchemaVersion); + var payload = dtoRecord.Payload; + Assert.Equal("APSB25-85", payload.GetValue("advisoryId").AsString); + Assert.Equal("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", payload.GetValue("detailUrl").AsString); + + var products = payload.GetValue("products").AsBsonArray.Select(static x => x.AsString).ToArray(); + Assert.Contains("Security update available for Adobe Acrobat Reader", products); + + var state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = state!.Cursor; + Assert.True(!cursor.TryGetValue("pendingDocuments", out _) + || cursor.GetValue("pendingDocuments").AsBsonArray.Count == 0); + Assert.True(!cursor.TryGetValue("pendingMappings", out _) + || cursor.GetValue("pendingMappings").AsBsonArray.Count == 0); + + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var acrobatAdvisory = advisories.Single(a => a.AdvisoryKey == "APSB25-85"); + Assert.Contains("APSB25-85", acrobatAdvisory.Aliases); + Assert.Equal( + acrobatAdvisory.References.Select(static r => r.Url).Distinct(StringComparer.OrdinalIgnoreCase).Count(), + acrobatAdvisory.References.Length); + + var premiereAdvisory = advisories.Single(a => a.AdvisoryKey == "APSB25-87"); + Assert.Contains("APSB25-87", premiereAdvisory.Aliases); + Assert.Equal( + premiereAdvisory.References.Select(static r => r.Url).Distinct(StringComparer.OrdinalIgnoreCase).Count(), + premiereAdvisory.References.Length); + + var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); + var snapshot = SnapshotSerializer.ToSnapshot(ordered); + var expected = ReadFixture("adobe-advisories.snapshot.json"); + var normalizedSnapshot = NormalizeLineEndings(snapshot); + var normalizedExpected = NormalizeLineEndings(expected); + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Adobe", "Fixtures", "adobe-advisories.actual.json"); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + + var flagsCollection = _fixture.Database.GetCollection("psirt_flags"); + var rawFlags = await flagsCollection.Find(Builders.Filter.Empty).ToListAsync(); + Assert.NotEmpty(rawFlags); + + var flagRecord = rawFlags.Single(doc => doc["_id"].AsString == "APSB25-87"); + Assert.Equal("Adobe", flagRecord["vendor"].AsString); + } + + [Fact] + public async Task Fetch_WithNotModifiedResponses_KeepsDocumentsMapped() + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler); + SeedIndex(handler); + SeedDetail(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var acrobatDoc = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", + CancellationToken.None); + Assert.NotNull(acrobatDoc); + Assert.Equal(DocumentStatuses.Mapped, acrobatDoc!.Status); + + var premiereDoc = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html", + CancellationToken.None); + Assert.NotNull(premiereDoc); + Assert.Equal(DocumentStatuses.Mapped, premiereDoc!.Status); + + SeedIndex(handler); + SeedDetailNotModified(handler); + + await connector.FetchAsync(provider, CancellationToken.None); + + acrobatDoc = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/acrobat/apsb25-85.html", + CancellationToken.None); + Assert.NotNull(acrobatDoc); + Assert.Equal(DocumentStatuses.Mapped, acrobatDoc!.Status); + + premiereDoc = await documentStore.FindBySourceAndUriAsync( + VndrAdobeConnectorPlugin.SourceName, + "https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html", + CancellationToken.None); + Assert.NotNull(premiereDoc); + Assert.Equal(DocumentStatuses.Mapped, premiereDoc!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrAdobeConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs) && pendingDocs.AsBsonArray.Count == 0); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMap) && pendingMap.AsBsonArray.Count == 0); + } + + private async Task BuildServiceProviderAsync(CannedHttpMessageHandler handler) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddAdobeConnector(opts => + { + opts.IndexUri = new Uri("https://helpx.adobe.com/security/security-bulletin.html"); + opts.InitialBackfill = TimeSpan.FromDays(30); + opts.WindowOverlap = TimeSpan.FromDays(2); + }); + + services.Configure(AdobeOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private static void SeedIndex(CannedHttpMessageHandler handler) + { + var indexUri = new Uri("https://helpx.adobe.com/security/security-bulletin.html"); + var indexHtml = ReadFixture("adobe-index.html"); + handler.AddTextResponse(indexUri, indexHtml, "text/html"); + } + + private static void SeedDetail(CannedHttpMessageHandler handler) + { + AddDetailResponse( + handler, + new Uri("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html"), + "adobe-detail-apsb25-85.html", + "\"apsb25-85\""); + + AddDetailResponse( + handler, + new Uri("https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html"), + "adobe-detail-apsb25-87.html", + "\"apsb25-87\""); + } + + private static void SeedDetailNotModified(CannedHttpMessageHandler handler) + { + AddNotModifiedResponse( + handler, + new Uri("https://helpx.adobe.com/security/products/acrobat/apsb25-85.html"), + "\"apsb25-85\""); + + AddNotModifiedResponse( + handler, + new Uri("https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html"), + "\"apsb25-87\""); + } + + private static void AddDetailResponse(CannedHttpMessageHandler handler, Uri uri, string fixture, string? etag) + { + handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static void AddNotModifiedResponse(CannedHttpMessageHandler handler, Uri uri, string? etag) + { + handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static List ExtractGuidList(BsonDocument cursor, string field) + { + if (!cursor.TryGetValue(field, out var value) || value is not BsonArray array) + { + return new List(); + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.AsString, out var guid)) + { + list.Add(guid); + } + } + return list; + } + + private static string ReadFixture(string name) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Adobe", "Fixtures", name); + return File.ReadAllText(path); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json new file mode 100644 index 00000000..bef21a2d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json @@ -0,0 +1,108 @@ +[ + { + "advisoryKey": "APSB25-85", + "affectedPackages": [ + { + "identifier": "Security update available for Adobe Acrobat Reader", + "platform": null, + "provenance": [ + { + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-85" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [] + } + ], + "aliases": [ + "APSB25-85" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-85" + } + ], + "published": "2025-09-09T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-85" + }, + "sourceTag": "adobe-psirt", + "summary": "Date published: September 9, 2025", + "url": "https://helpx.adobe.com/security/products/acrobat/apsb25-85.html" + } + ], + "severity": null, + "summary": "Date published: September 9, 2025", + "title": "APSB25-85: Security update available for Adobe Acrobat Reader" + }, + { + "advisoryKey": "APSB25-87", + "affectedPackages": [ + { + "identifier": "Security update available for Adobe Premiere Pro", + "platform": null, + "provenance": [ + { + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-87" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [] + } + ], + "aliases": [ + "APSB25-87" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-87" + } + ], + "published": "2025-09-08T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "kind": "parser", + "recordedAt": "2025-09-10T00:00:00+00:00", + "source": "vndr-adobe", + "value": "APSB25-87" + }, + "sourceTag": "adobe-psirt", + "summary": "Date published: September 8, 2025", + "url": "https://helpx.adobe.com/security/products/premiere_pro/apsb25-87.html" + } + ], + "severity": null, + "summary": "Date published: September 8, 2025", + "title": "APSB25-87: Security update available for Adobe Premiere Pro" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html new file mode 100644 index 00000000..fce3db2c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html @@ -0,0 +1,10 @@ + + + + APSB25-85 + + +

    APSB25-85 Security update available for Adobe Acrobat Reader

    +

    Date published: September 9, 2025

    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html new file mode 100644 index 00000000..eef7154a --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html @@ -0,0 +1,10 @@ + + + + APSB25-87 + + +

    APSB25-87 Security update available for Adobe Premiere Pro

    +

    Date published: September 8, 2025

    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html new file mode 100644 index 00000000..f859eae9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + +
    APSB25-85: Security update available for Adobe Acrobat ReaderSeptember 9, 2025
    APSB25-87: Security update available for Adobe Premiere Pro09/08/2025
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj new file mode 100644 index 00000000..d05dbb40 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md new file mode 100644 index 00000000..29038dd5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Adobe PSIRT connector ingesting APSB/APA advisories; authoritative for Adobe products; emits psirt_flags and affected ranges; establishes PSIRT precedence over registry or distro data for Adobe software. +## Scope +- Discover and fetch APSB/APA index and detail pages; follow product links as needed; window by advisory ID/date. +- Validate HTML or JSON; normalize titles, CVE lists, product components, fixed versions/builds; capture mitigation notes and KBs. +- Persist raw docs with sha256 and headers; maintain source_state cursors; ensure idempotent mapping. +## Participants +- Source.Common (HTTP, HTML parsing, retries/backoff, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). +- Models (canonical Advisory/Affected/Provenance). +- Core/WebService (jobs: source:adobe:fetch|parse|map). +- Merge engine (later) to apply PSIRT override policy for Adobe packages. +## Interfaces & contracts +- Aliases include APSB-YYYY-XX (and APA-* when present) plus CVE ids. +- Affected entries capture Vendor=Adobe, Product/component names, Type=vendor, Identifier stable (for example product slug), Versions with fixed/fixedBy where available. +- References typed: advisory, patch, mitigation, release notes; URLs normalized and deduped. +- Provenance.method="parser"; value carries advisory id and URL; recordedAt=fetch time. +## In/Out of scope +In: PSIRT ingestion, aliases, affected plus fixedBy, psirt_flags, watermark/resume. +Out: signing, package artifact downloads, non-Adobe product truth. +## Observability & security expectations +- Metrics: adobe.fetch.count, adobe.parse.fail, adobe.map.affected_count, adobe.cursor.moves. +- Logs: advisory ids, product counts, extraction timings; hosts allowlisted; no secret logging. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Adobe.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs new file mode 100644 index 00000000..0745bf37 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs @@ -0,0 +1,503 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Json.Schema; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; +using StellaOps.Feedser.Source.Vndr.Adobe.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Models; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Adobe; + +public sealed class AdobeConnector : IFeedConnector +{ + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly IJsonSchemaValidator _schemaValidator; + private readonly AdobeOptions _options; + private readonly TimeProvider _timeProvider; + private readonly IHttpClientFactory _httpClientFactory; + private readonly AdobeDiagnostics _diagnostics; + private readonly ILogger _logger; + + private static readonly JsonSchema Schema = AdobeSchemaProvider.Schema; + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + public AdobeConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IPsirtFlagStore psirtFlagStore, + IJsonSchemaValidator schemaValidator, + IOptions options, + TimeProvider? timeProvider, + IHttpClientFactory httpClientFactory, + AdobeDiagnostics diagnostics, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VndrAdobeConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var backfillStart = now - _options.InitialBackfill; + var windowStart = cursor.LastPublished.HasValue + ? cursor.LastPublished.Value - _options.WindowOverlap + : backfillStart; + if (windowStart < backfillStart) + { + windowStart = backfillStart; + } + + var maxPublished = cursor.LastPublished; + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + var fetchCache = cursor.FetchCache is null + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(cursor.FetchCache, StringComparer.Ordinal); + var touchedResources = new HashSet(StringComparer.Ordinal); + + var collectedEntries = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var indexUri in EnumerateIndexUris()) + { + _diagnostics.FetchAttempt(); + string? html = null; + try + { + var client = _httpClientFactory.CreateClient(AdobeOptions.HttpClientName); + using var response = await client.GetAsync(indexUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + html = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Failed to download Adobe index page {Uri}", indexUri); + continue; + } + + if (string.IsNullOrEmpty(html)) + { + continue; + } + + IReadOnlyCollection entries; + try + { + entries = AdobeIndexParser.Parse(html, indexUri); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to parse Adobe index page {Uri}", indexUri); + _diagnostics.FetchFailure(); + continue; + } + + foreach (var entry in entries) + { + if (entry.PublishedUtc < windowStart) + { + continue; + } + + if (!collectedEntries.TryGetValue(entry.AdvisoryId, out var existing) || entry.PublishedUtc > existing.PublishedUtc) + { + collectedEntries[entry.AdvisoryId] = entry; + } + } + } + + foreach (var entry in collectedEntries.Values.OrderBy(static e => e.PublishedUtc)) + { + if (!maxPublished.HasValue || entry.PublishedUtc > maxPublished) + { + maxPublished = entry.PublishedUtc; + } + + var cacheKey = entry.DetailUri.ToString(); + touchedResources.Add(cacheKey); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["advisoryId"] = entry.AdvisoryId, + ["published"] = entry.PublishedUtc.ToString("O"), + ["title"] = entry.Title ?? string.Empty, + }; + + try + { + var result = await _fetchService.FetchAsync( + new SourceFetchRequest(AdobeOptions.HttpClientName, SourceName, entry.DetailUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }, + cancellationToken).ConfigureAwait(false); + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (cursor.TryGetFetchCache(cacheKey, out var cached) + && string.Equals(cached.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase)) + { + _diagnostics.FetchUnchanged(); + fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256); + await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + continue; + } + + _diagnostics.FetchDocument(); + fetchCache[cacheKey] = new AdobeFetchCacheEntry(result.Document.Sha256); + + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Failed to fetch Adobe advisory {AdvisoryId} ({Uri})", entry.AdvisoryId, entry.DetailUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + foreach (var key in fetchCache.Keys.ToList()) + { + if (!touchedResources.Contains(key)) + { + fetchCache.Remove(key); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastPublished(maxPublished) + .WithFetchCache(fetchCache); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Adobe document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + AdobeDocumentMetadata metadata; + try + { + metadata = AdobeDocumentMetadata.FromDocument(document); + } + catch (Exception ex) + { + _logger.LogError(ex, "Adobe metadata parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + AdobeBulletinDto dto; + try + { + var bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = Encoding.UTF8.GetString(bytes); + dto = AdobeDetailParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Adobe parse failed for advisory {AdvisoryId} ({Uri})", metadata.AdvisoryId, document.Uri); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + using var jsonDocument = JsonDocument.Parse(json); + _schemaValidator.Validate(jsonDocument, Schema, metadata.AdvisoryId); + + var payload = MongoDB.Bson.BsonDocument.Parse(json); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "adobe.bulletin.v1", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + var now = _timeProvider.GetUtcNow(); + + foreach (var documentId in cursor.PendingMappings) + { + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + AdobeBulletinDto? dto; + try + { + var json = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + dto = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Adobe DTO deserialization failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("Adobe DTO payload deserialized as null for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = BuildAdvisory(dto, now); + if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) + { + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + + var flag = new PsirtFlagRecord( + advisory.AdvisoryKey, + "Adobe", + SourceName, + dto.AdvisoryId, + now); + + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + } + else + { + _logger.LogWarning("Skipping PSIRT flag for advisory with missing key (document {DocumentId})", documentId); + } + + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private IEnumerable EnumerateIndexUris() + { + yield return _options.IndexUri; + foreach (var uri in _options.AdditionalIndexUris) + { + yield return uri; + } + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return AdobeCursor.FromBsonDocument(record?.Cursor); + } + + private async Task UpdateCursorAsync(AdobeCursor cursor, CancellationToken cancellationToken) + { + var updatedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), updatedAt, cancellationToken).ConfigureAwait(false); + } + + private Advisory BuildAdvisory(AdobeBulletinDto dto, DateTimeOffset recordedAt) + { + var provenance = new AdvisoryProvenance(SourceName, "parser", dto.AdvisoryId, recordedAt); + + var aliasSet = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId, + }; + foreach (var cve in dto.Cves) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliasSet.Add(cve); + } + } + + var comparer = StringComparer.OrdinalIgnoreCase; + var references = new List<(AdvisoryReference Reference, int Priority)> + { + (new AdvisoryReference(dto.DetailUrl, "advisory", "adobe-psirt", dto.Summary, provenance), 0), + }; + + foreach (var cve in dto.Cves) + { + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + var url = $"https://www.cve.org/CVERecord?id={cve}"; + references.Add((new AdvisoryReference(url, "advisory", cve, null, provenance), 1)); + } + + var orderedReferences = references + .GroupBy(tuple => tuple.Reference.Url, comparer) + .Select(group => group + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .First()) + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .Select(t => t.Reference) + .ToArray(); + + var affected = dto.Products.Select(product => + new AffectedPackage( + AffectedPackageTypes.Vendor, + NormalizeIdentifier(product), + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: new[] { provenance })) + .ToArray(); + + var aliases = aliasSet + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Select(static alias => alias.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static alias => alias, StringComparer.Ordinal) + .ToArray(); + + return new Advisory( + dto.AdvisoryId, + dto.Title, + dto.Summary, + language: "en", + published: dto.Published, + modified: null, + severity: null, + exploitKnown: false, + aliases, + orderedReferences, + affected, + Array.Empty(), + new[] { provenance }); + } + + private static string NormalizeIdentifier(string product) + { + if (string.IsNullOrWhiteSpace(product)) + { + return "Adobe Product"; + } + + return product.Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs new file mode 100644 index 00000000..eb980aff --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Adobe; + +public sealed class VndrAdobeConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-adobe"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs new file mode 100644 index 00000000..1cdf4e08 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs @@ -0,0 +1,49 @@ +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Vndr.Adobe; + +public sealed class AdobeDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Vndr.Adobe"; + private static readonly string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + + public AdobeDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter( + name: "adobe.fetch.attempts", + unit: "operations", + description: "Number of Adobe index fetch operations."); + _fetchDocuments = _meter.CreateCounter( + name: "adobe.fetch.documents", + unit: "documents", + description: "Number of Adobe advisory documents captured."); + _fetchFailures = _meter.CreateCounter( + name: "adobe.fetch.failures", + unit: "operations", + description: "Number of Adobe fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "adobe.fetch.unchanged", + unit: "documents", + description: "Number of Adobe advisories skipped due to unchanged content."); + } + + public Meter Meter => _meter; + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs new file mode 100644 index 00000000..0708055f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Adobe; + +public static class AdobeServiceCollectionExtensions +{ + public static IServiceCollection AddAdobeConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(AdobeOptions.HttpClientName, static (sp, options) => + { + var adobeOptions = sp.GetRequiredService>().Value; + options.BaseAddress = adobeOptions.IndexUri; + options.UserAgent = "StellaOps.Feedser.VndrAdobe/1.0"; + options.Timeout = TimeSpan.FromSeconds(20); + options.AllowedHosts.Clear(); + options.AllowedHosts.Add(adobeOptions.IndexUri.Host); + foreach (var additional in adobeOptions.AdditionalIndexUris) + { + options.AllowedHosts.Add(additional.Host); + } + }); + + services.TryAddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs new file mode 100644 index 00000000..3adba391 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Configuration; + +public sealed class AdobeOptions +{ + public const string HttpClientName = "source-vndr-adobe"; + + public Uri IndexUri { get; set; } = new("https://helpx.adobe.com/security/security-bulletin.html"); + + public List AdditionalIndexUris { get; } = new(); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90); + + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(3); + + public int MaxEntriesPerFetch { get; set; } = 100; + + public void Validate() + { + if (IndexUri is null || !IndexUri.IsAbsoluteUri) + { + throw new ArgumentException("IndexUri must be an absolute URI.", nameof(IndexUri)); + } + + foreach (var uri in AdditionalIndexUris) + { + if (uri is null || !uri.IsAbsoluteUri) + { + throw new ArgumentException("Additional index URIs must be absolute.", nameof(AdditionalIndexUris)); + } + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new ArgumentException("InitialBackfill must be positive.", nameof(InitialBackfill)); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new ArgumentException("WindowOverlap cannot be negative.", nameof(WindowOverlap)); + } + + if (MaxEntriesPerFetch <= 0) + { + throw new ArgumentException("MaxEntriesPerFetch must be positive.", nameof(MaxEntriesPerFetch)); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs new file mode 100644 index 00000000..40209c9c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal sealed record AdobeBulletinDto( + string AdvisoryId, + string Title, + DateTimeOffset Published, + IReadOnlyList Products, + IReadOnlyList Cves, + string DetailUrl, + string? Summary) +{ + public static AdobeBulletinDto Create( + string advisoryId, + string title, + DateTimeOffset published, + IEnumerable? products, + IEnumerable? cves, + Uri detailUri, + string? summary) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryId); + ArgumentException.ThrowIfNullOrEmpty(title); + ArgumentNullException.ThrowIfNull(detailUri); + + var productList = products?.Where(static p => !string.IsNullOrWhiteSpace(p)) + .Select(static p => p.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase) + .ToList() ?? new List(); + + var cveList = cves?.Where(static c => !string.IsNullOrWhiteSpace(c)) + .Select(static c => c.Trim().ToUpperInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static c => c, StringComparer.Ordinal) + .ToList() ?? new List(); + + return new AdobeBulletinDto( + advisoryId.ToUpperInvariant(), + title.Trim(), + published.ToUniversalTime(), + productList, + cveList, + detailUri.ToString(), + string.IsNullOrWhiteSpace(summary) ? null : summary.Trim()); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs new file mode 100644 index 00000000..19321bb2 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal sealed record AdobeCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary? FetchCache) +{ + public static AdobeCursor Empty { get; } = new(null, Array.Empty(), Array.Empty(), null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); + + if (FetchCache is { Count: > 0 }) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBson(); + } + + document["fetchCache"] = cacheDocument; + } + + return document; + } + + public static AdobeCursor FromBsonDocument(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastPublished = null; + if (document.TryGetValue("lastPublished", out var lastPublishedValue)) + { + lastPublished = ReadDateTime(lastPublishedValue); + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new AdobeCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache); + } + + public AdobeCursor WithLastPublished(DateTimeOffset? value) + => this with { LastPublished = value?.ToUniversalTime() }; + + public AdobeCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public AdobeCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public AdobeCursor WithFetchCache(IDictionary? cache) + { + if (cache is null) + { + return this with { FetchCache = null }; + } + + var target = new Dictionary(cache, StringComparer.Ordinal); + return this with { FetchCache = target }; + } + + public bool TryGetFetchCache(string key, out AdobeFetchCacheEntry entry) + { + var cache = FetchCache; + if (cache is null) + { + entry = AdobeFetchCacheEntry.Empty; + return false; + } + + if (cache.TryGetValue(key, out var value) && value is not null) + { + entry = value; + return true; + } + + entry = AdobeFetchCacheEntry.Empty; + return false; + } + + private static DateTimeOffset? ReadDateTime(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary? ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument) + { + return null; + } + + var dictionary = new Dictionary(StringComparer.Ordinal); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entryDocument) + { + dictionary[element.Name] = AdobeFetchCacheEntry.FromBson(entryDocument); + } + } + + return dictionary; + } +} + +internal sealed record AdobeFetchCacheEntry(string Sha256) +{ + public static AdobeFetchCacheEntry Empty { get; } = new(string.Empty); + + public BsonDocument ToBson() + { + var document = new BsonDocument + { + ["sha256"] = Sha256, + }; + + return document; + } + + public static AdobeFetchCacheEntry FromBson(BsonDocument document) + { + var sha = document.TryGetValue("sha256", out var shaValue) ? shaValue.AsString : string.Empty; + return new AdobeFetchCacheEntry(sha); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs new file mode 100644 index 00000000..0ff1b2d9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal static class AdobeDetailParser +{ + private static readonly HtmlParser Parser = new(); + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly string[] DateMarkers = { "date published", "release date", "published" }; + + public static AdobeBulletinDto Parse(string html, AdobeDocumentMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + using var document = Parser.ParseDocument(html); + var title = metadata.Title ?? document.QuerySelector("h1")?.TextContent?.Trim() ?? metadata.AdvisoryId; + var summary = document.QuerySelector("p")?.TextContent?.Trim(); + + var published = metadata.PublishedUtc ?? TryExtractPublished(document) ?? DateTimeOffset.UtcNow; + + var cves = ExtractCves(document.Body?.TextContent ?? string.Empty); + var products = ExtractProducts(title, document); + + return AdobeBulletinDto.Create( + metadata.AdvisoryId, + title, + published, + products, + cves, + metadata.DetailUri, + summary); + } + + private static IEnumerable ExtractCves(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + yield break; + } + + foreach (Match match in CveRegex.Matches(text)) + { + yield return match.Value; + } + } + + private static IEnumerable ExtractProducts(string title, IDocument document) + { + var products = new List(); + + if (!string.IsNullOrWhiteSpace(title)) + { + var split = title.Split(':', 2, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (split.Length == 2) + { + products.Add(split[1]); + } + else if (split.Length == 1) + { + var parts = title.Split(" for ", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 2) + { + products.Add(parts[1]); + } + } + } + + var tableProducts = document.QuerySelectorAll("table td") + .Select(cell => cell.TextContent?.Trim()) + .Where(text => !string.IsNullOrWhiteSpace(text) && text!.Contains("Adobe", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + products.AddRange(tableProducts!); + return products; + } + + private static DateTimeOffset? TryExtractPublished(IDocument document) + { + var candidates = new List(); + candidates.Add(document.QuerySelector("time")?.GetAttribute("datetime")); + candidates.Add(document.QuerySelector("time")?.TextContent); + + foreach (var marker in DateMarkers) + { + var element = document.All.FirstOrDefault(node => node.TextContent.Contains(marker, StringComparison.OrdinalIgnoreCase)); + if (element is not null) + { + candidates.Add(element.TextContent); + } + } + + foreach (var candidate in candidates) + { + if (TryParseDate(candidate, out var parsed)) + { + return parsed; + } + } + + return null; + } + + private static bool TryParseDate(string? value, out DateTimeOffset result) + { + result = default; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out result)) + { + result = result.ToUniversalTime(); + return true; + } + + if (DateTime.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)) + { + result = new DateTimeOffset(date, TimeSpan.Zero).ToUniversalTime(); + return true; + } + + return false; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs new file mode 100644 index 00000000..6283f605 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal sealed record AdobeDocumentMetadata( + string AdvisoryId, + string? Title, + DateTimeOffset? PublishedUtc, + Uri DetailUri) +{ + private const string AdvisoryIdKey = "advisoryId"; + private const string TitleKey = "title"; + private const string PublishedKey = "published"; + + public static AdobeDocumentMetadata FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + if (document.Metadata is null) + { + throw new InvalidOperationException("Adobe document metadata is missing."); + } + + var advisoryId = document.Metadata.TryGetValue(AdvisoryIdKey, out var idValue) ? idValue : null; + if (string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("Adobe document advisoryId metadata missing."); + } + + var title = document.Metadata.TryGetValue(TitleKey, out var titleValue) ? titleValue : null; + DateTimeOffset? published = null; + if (document.Metadata.TryGetValue(PublishedKey, out var publishedValue) + && DateTimeOffset.TryParse(publishedValue, out var parsedPublished)) + { + published = parsedPublished.ToUniversalTime(); + } + + if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri)) + { + throw new InvalidOperationException("Adobe document URI invalid."); + } + + return new AdobeDocumentMetadata(advisoryId.Trim(), string.IsNullOrWhiteSpace(title) ? null : title.Trim(), published, detailUri); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs new file mode 100644 index 00000000..961d00d5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs @@ -0,0 +1,5 @@ +using System; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal sealed record AdobeIndexEntry(string AdvisoryId, Uri DetailUri, DateTimeOffset PublishedUtc, string? Title); diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs new file mode 100644 index 00000000..ad479a8d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal static class AdobeIndexParser +{ + private static readonly HtmlParser Parser = new(); + private static readonly Regex AdvisoryIdRegex = new("(APSB|APA)\\d{2}-\\d{2,}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly string[] ExplicitFormats = + { + "MMMM d, yyyy", + "MMM d, yyyy", + "M/d/yyyy", + "MM/dd/yyyy", + "yyyy-MM-dd", + }; + + public static IReadOnlyCollection Parse(string html, Uri baseUri) + { + ArgumentNullException.ThrowIfNull(html); + ArgumentNullException.ThrowIfNull(baseUri); + + var document = Parser.ParseDocument(html); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var anchors = document.QuerySelectorAll("a[href]"); + + foreach (var anchor in anchors) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!href.Contains("/security/products/", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!TryExtractAdvisoryId(anchor.TextContent, href, out var advisoryId)) + { + continue; + } + + if (!Uri.TryCreate(baseUri, href, out var detailUri)) + { + continue; + } + + var published = TryResolvePublished(anchor) ?? DateTimeOffset.UtcNow; + var entry = new AdobeIndexEntry(advisoryId.ToUpperInvariant(), detailUri, published, anchor.TextContent?.Trim()); + map[entry.AdvisoryId] = entry; + } + + return map.Values + .OrderBy(static e => e.PublishedUtc) + .ThenBy(static e => e.AdvisoryId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static bool TryExtractAdvisoryId(string? text, string href, out string advisoryId) + { + if (!string.IsNullOrWhiteSpace(text)) + { + var match = AdvisoryIdRegex.Match(text); + if (match.Success) + { + advisoryId = match.Value.ToUpperInvariant(); + return true; + } + } + + var hrefMatch = AdvisoryIdRegex.Match(href); + if (hrefMatch.Success) + { + advisoryId = hrefMatch.Value.ToUpperInvariant(); + return true; + } + + advisoryId = string.Empty; + return false; + } + + private static DateTimeOffset? TryResolvePublished(IElement anchor) + { + var row = anchor.Closest("tr"); + if (row is not null) + { + var cells = row.GetElementsByTagName("td"); + if (cells.Length >= 2) + { + for (var idx = 1; idx < cells.Length; idx++) + { + if (TryParseDate(cells[idx].TextContent, out var parsed)) + { + return parsed; + } + } + } + } + + var sibling = anchor.NextElementSibling; + while (sibling is not null) + { + if (TryParseDate(sibling.TextContent, out var parsed)) + { + return parsed; + } + + sibling = sibling.NextElementSibling; + } + + if (TryParseDate(anchor.ParentElement?.TextContent, out var parentDate)) + { + return parentDate; + } + + return null; + } + + private static bool TryParseDate(string? value, out DateTimeOffset result) + { + result = default; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out result)) + { + return Normalize(ref result); + } + + foreach (var format in ExplicitFormats) + { + if (DateTime.TryParseExact(trimmed, format, CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)) + { + result = new DateTimeOffset(date, TimeSpan.Zero); + return Normalize(ref result); + } + } + + return false; + } + + private static bool Normalize(ref DateTimeOffset value) + { + value = value.ToUniversalTime(); + value = new DateTimeOffset(value.Year, value.Month, value.Day, 0, 0, 0, TimeSpan.Zero); + return true; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs new file mode 100644 index 00000000..40fa1ac9 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs @@ -0,0 +1,25 @@ +using System.IO; +using System.Reflection; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; + +internal static class AdobeSchemaProvider +{ + private static readonly Lazy Cached = new(Load, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => Cached.Value; + + private static JsonSchema Load() + { + var assembly = typeof(AdobeSchemaProvider).GetTypeInfo().Assembly; + const string resourceName = "StellaOps.Feedser.Source.Vndr.Adobe.Schemas.adobe-bulletin.schema.json"; + + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); + using var reader = new StreamReader(stream); + var schemaText = reader.ReadToEnd(); + return JsonSchema.FromText(schemaText); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json b/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json new file mode 100644 index 00000000..e000460e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://stellaops.example/schemas/adobe-bulletin.schema.json", + "type": "object", + "required": [ + "advisoryId", + "title", + "published", + "products", + "cves", + "detailUrl" + ], + "properties": { + "advisoryId": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string", + "minLength": 1 + }, + "published": { + "type": "string", + "format": "date-time" + }, + "products": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "cves": { + "type": "array", + "items": { + "type": "string", + "pattern": "^CVE-\\d{4}-\\d{4,}$" + } + }, + "detailUrl": { + "type": "string", + "format": "uri" + }, + "summary": { + "type": ["string", "null"] + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj b/src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj new file mode 100644 index 00000000..89b5f59e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md new file mode 100644 index 00000000..4efacf27 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md @@ -0,0 +1,11 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Index discovery and sliding window fetch|BE-Conn-Adobe|Source.Common|DONE — Support backfill; honor robots/ToS.| +|Detail extractor (products/components/fixes)|BE-Conn-Adobe|Source.Common|DONE — Normalizes metadata and CVE/product capture.| +|DTO schema and validation pipeline|BE-Conn-Adobe, QA|Source.Common|DONE — JSON schema enforced during parse.| +|Canonical mapping plus psirt_flags|BE-Conn-Adobe|Models|DONE — Emits canonical advisory and Adobe psirt flag.| +|SourceState plus sha256 short-circuit|BE-Conn-Adobe|Storage.Mongo|DONE — Idempotence guarantee.| +|Golden fixtures and determinism tests|QA|Source.Vndr.Adobe|**DONE** — connector tests assert snapshot determinism for dual advisories.| +|Mark failed parse DTOs|BE-Conn-Adobe|Storage.Mongo|**DONE** — parse failures now mark documents `Failed` and tests cover the path.| +|Reference dedupe & ordering|BE-Conn-Adobe|Models|**DONE** — mapper groups references by URL with deterministic ordering.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Class1.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Class1.cs new file mode 100644 index 00000000..758b2125 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Apple; + +public sealed class VndrAppleConnectorPlugin : IConnectorPlugin +{ + public string Name => "vndr-apple"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj b/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs new file mode 100644 index 00000000..7f537256 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs @@ -0,0 +1,347 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Vndr.Chromium; +using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Tests; + +[Collection("mongo-fixture")] +public sealed class ChromiumConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly List _allocatedDatabases = new(); + + public ChromiumConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 10, 18, 0, 0, TimeSpan.Zero)); + } + + [Fact] + public async Task FetchParseMap_ProducesSnapshot() + { + var databaseName = AllocateDatabaseName(); + await DropDatabaseAsync(databaseName); + + try + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler, databaseName); + SeedHttpFixtures(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + try + { + await connector.ParseAsync(provider, CancellationToken.None); + } + catch (StellaOps.Feedser.Source.Common.Json.JsonSchemaValidationException) + { + // Parsing should flag document as failed even when schema validation rejects payloads. + } + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var advisory = Assert.Single(advisories); + + Assert.Equal("chromium/post/stable-channel-update-for-desktop", advisory.AdvisoryKey); + Assert.Contains("CHROMIUM-POST:stable-channel-update-for-desktop", advisory.Aliases); + Assert.Contains("CVE-2024-12345", advisory.Aliases); + Assert.Contains("CVE-2024-22222", advisory.Aliases); + + Assert.Contains(advisory.AffectedPackages, package => package.Platform == "android" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.89")); + Assert.Contains(advisory.AffectedPackages, package => package.Platform == "linux" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.137")); + Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138")); + Assert.Contains(advisory.AffectedPackages, package => package.Identifier == "google:chrome:extended-stable" && package.Platform == "windows-mac" && package.VersionRanges.Any(range => range.FixedVersion == "128.0.6613.138")); + + Assert.Contains(advisory.References, reference => reference.Url.Contains("chromium.googlesource.com", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(advisory.References, reference => reference.Url.Contains("issues.chromium.org", StringComparison.OrdinalIgnoreCase)); + + var psirtStore = provider.GetRequiredService(); + var psirtFlag = await psirtStore.FindAsync(advisory.AdvisoryKey, CancellationToken.None); + Assert.NotNull(psirtFlag); + Assert.Equal("Google", psirtFlag!.Vendor); + + var canonicalJson = CanonicalJsonSerializer.Serialize(advisory).Trim(); + var snapshotPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Chromium", "Fixtures", "chromium-advisory.snapshot.json"); + var expected = File.ReadAllText(snapshotPath).Trim(); + if (!string.Equals(expected, canonicalJson, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Chromium", "Fixtures", "chromium-advisory.actual.json"); + File.WriteAllText(actualPath, canonicalJson); + } + Assert.Equal(expected, canonicalJson); + } + finally + { + await DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task ParseFailure_MarksDocumentFailed() + { + var databaseName = AllocateDatabaseName(); + await DropDatabaseAsync(databaseName); + + try + { + var handler = new CannedHttpMessageHandler(); + var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false"); + var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"); + + handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml"); + handler.AddTextResponse(detailUri, "
    missing post body
    ", "text/html"); + + await using var provider = await BuildServiceProviderAsync(handler, databaseName); + var connector = provider.GetRequiredService(); + + await connector.FetchAsync(provider, CancellationToken.None); + try + { + await connector.ParseAsync(provider, CancellationToken.None); + } + catch (JsonSchemaValidationException) + { + // Expected for malformed posts; connector should still flag the document as failed. + } + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(VndrChromiumConnectorPlugin.SourceName, detailUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Failed, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocuments); + } + finally + { + await DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task Resume_CompletesPendingDocumentsAfterRestart() + { + var databaseName = AllocateDatabaseName(); + await DropDatabaseAsync(databaseName); + + try + { + var fetchHandler = new CannedHttpMessageHandler(); + Guid[] pendingDocumentIds; + await using (var fetchProvider = await BuildServiceProviderAsync(fetchHandler, databaseName)) + { + SeedHttpFixtures(fetchHandler); + var connector = fetchProvider.GetRequiredService(); + await connector.FetchAsync(fetchProvider, CancellationToken.None); + + var stateRepository = fetchProvider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.NotEmpty(pendingDocuments); + pendingDocumentIds = pendingDocuments.Select(value => Guid.Parse(value.AsString)).ToArray(); + } + + var resumeHandler = new CannedHttpMessageHandler(); + SeedHttpFixtures(resumeHandler); + await using var resumeProvider = await BuildServiceProviderAsync(resumeHandler, databaseName); + var stateRepositoryBefore = resumeProvider.GetRequiredService(); + var resumeState = await stateRepositoryBefore.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(resumeState); + var resumePendingDocs = resumeState!.Cursor.TryGetValue("pendingDocuments", out var resumePendingValue) + ? resumePendingValue.AsBsonArray + : new BsonArray(); + Assert.Equal(pendingDocumentIds.Length, resumePendingDocs.Count); + var resumeIds = resumePendingDocs.Select(value => Guid.Parse(value.AsString)).OrderBy(id => id).ToArray(); + Assert.Equal(pendingDocumentIds.OrderBy(id => id).ToArray(), resumeIds); + + var resumeConnector = resumeProvider.GetRequiredService(); + await resumeConnector.ParseAsync(resumeProvider, CancellationToken.None); + await resumeConnector.MapAsync(resumeProvider, CancellationToken.None); + + var documentStore = resumeProvider.GetRequiredService(); + foreach (var documentId in pendingDocumentIds) + { + var document = await documentStore.FindAsync(documentId, CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + } + + var stateRepositoryAfter = resumeProvider.GetRequiredService(); + var finalState = await stateRepositoryAfter.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(finalState); + var finalPending = finalState!.Cursor.TryGetValue("pendingDocuments", out var finalPendingDocs) + ? finalPendingDocs.AsBsonArray + : new BsonArray(); + Assert.Empty(finalPending); + + var finalPendingMappings = finalState.Cursor.TryGetValue("pendingMappings", out var finalPendingMappingsValue) + ? finalPendingMappingsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(finalPendingMappings); + } + finally + { + await DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task Fetch_SkipsUnchangedDocuments() + { + var databaseName = AllocateDatabaseName(); + await DropDatabaseAsync(databaseName); + + try + { + var handler = new CannedHttpMessageHandler(); + await using var provider = await BuildServiceProviderAsync(handler, databaseName); + SeedHttpFixtures(handler); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Single(advisories); + + // Re-seed responses and fetch again with unchanged content. + SeedHttpFixtures(handler); + await connector.FetchAsync(provider, CancellationToken.None); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(VndrChromiumConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + var cursor = state!.Cursor; + var pendingDocuments = cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingDocuments); + + var pendingMappings = cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue.AsBsonArray + : new BsonArray(); + Assert.Empty(pendingMappings); + } + finally + { + await DropDatabaseAsync(databaseName); + } + } + + private async Task BuildServiceProviderAsync(CannedHttpMessageHandler handler, string databaseName) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = databaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddChromiumConnector(opts => + { + opts.FeedUri = new Uri("https://chromereleases.googleblog.com/atom.xml"); + opts.InitialBackfill = TimeSpan.FromDays(30); + opts.WindowOverlap = TimeSpan.FromDays(1); + opts.MaxFeedPages = 1; + opts.MaxEntriesPerPage = 50; + }); + + services.Configure(ChromiumOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + + return provider; + } + + private string AllocateDatabaseName() + { + var name = $"chromium-tests-{Guid.NewGuid():N}"; + _allocatedDatabases.Add(name); + return name; + } + + private async Task DropDatabaseAsync(string databaseName) + { + try + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + catch (MongoCommandException ex) when (ex.CodeName == "NamespaceNotFound") + { + } + } + + private static void SeedHttpFixtures(CannedHttpMessageHandler handler) + { + var feedUri = new Uri("https://chromereleases.googleblog.com/atom.xml?max-results=50&start-index=1&redirect=false"); + var detailUri = new Uri("https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"); + + handler.AddTextResponse(feedUri, ReadFixture("chromium-feed.xml"), "application/atom+xml"); + handler.AddTextResponse(detailUri, ReadFixture("chromium-detail.html"), "text/html"); + } + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Chromium", "Fixtures", filename); + return File.ReadAllText(path); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + foreach (var name in _allocatedDatabases.Distinct(StringComparer.Ordinal)) + { + await DropDatabaseAsync(name); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs new file mode 100644 index 00000000..4567ef56 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs @@ -0,0 +1,47 @@ +using System; +using System.Linq; +using StellaOps.Feedser.Source.Vndr.Chromium; +using StellaOps.Feedser.Source.Vndr.Chromium.Internal; +using Xunit; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Tests; + +public sealed class ChromiumMapperTests +{ + [Fact] + public void Map_DeduplicatesReferencesAndOrdersDeterministically() + { + var published = new DateTimeOffset(2024, 9, 12, 14, 0, 0, TimeSpan.Zero); + var metadata = new ChromiumDocumentMetadata( + "post-123", + "Stable Channel Update", + new Uri("https://chromium.example/stable-update.html"), + published, + null, + "Security fixes"); + + var dto = ChromiumDto.From( + metadata, + new[] { "CVE-2024-0001" }, + new[] { "windows" }, + new[] { new ChromiumVersionInfo("windows", "stable", "128.0.6613.88") }, + new[] + { + new ChromiumReference("https://chromium.example/ref1", "advisory", "Ref 1"), + new ChromiumReference("https://chromium.example/ref1", "advisory", "Ref 1 duplicate"), + new ChromiumReference("https://chromium.example/ref2", "patch", "Ref 2"), + }); + + var (advisory, _) = ChromiumMapper.Map(dto, VndrChromiumConnectorPlugin.SourceName, published); + + var referenceUrls = advisory.References.Select(r => r.Url).ToArray(); + Assert.Equal( + new[] + { + "https://chromium.example/stable-update.html", + "https://chromium.example/ref1", + "https://chromium.example/ref2", + }, + referenceUrls); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json new file mode 100644 index 00000000..bb737de8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json @@ -0,0 +1 @@ +{"advisoryKey":"chromium/post/stable-channel-update-for-desktop","affectedPackages":[{"identifier":"google:chrome","platform":"android","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.89","introducedVersion":null,"lastAffectedVersion":null,"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome","platform":"linux","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.137","introducedVersion":null,"lastAffectedVersion":null,"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome","platform":"windows-mac","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.138","introducedVersion":null,"lastAffectedVersion":null,"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]},{"identifier":"google:chrome:extended-stable","platform":"windows-mac","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"statuses":[],"type":"vendor","versionRanges":[{"fixedVersion":"128.0.6613.138","introducedVersion":null,"lastAffectedVersion":null,"provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"rangeExpression":null,"rangeKind":"vendor"}]}],"aliases":["CHROMIUM-POST:2024-09-10","CHROMIUM-POST:stable-channel-update-for-desktop","CVE-2024-12345","CVE-2024-22222"],"cvssMetrics":[],"exploitKnown":false,"language":"en","modified":"2024-09-10T17:45:00+00:00","provenance":[{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"}],"published":"2024-09-10T17:30:00+00:00","references":[{"kind":"advisory","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"chromium-blog","summary":null,"url":"https://chromereleases.googleblog.com/2024/09/stable-channel-update-for-desktop.html"},{"kind":"changelog","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"changelog","summary":"log","url":"https://chromium.googlesource.com/chromium/src/+log/128.0.6613.120..128.0.6613.138"},{"kind":"doc","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"doc","summary":"security page","url":"https://chromium.org/Home/chromium-security"},{"kind":"bug","provenance":{"kind":"document","recordedAt":"2024-09-10T18:00:00+00:00","source":"vndr-chromium","value":"stable-channel-update-for-desktop"},"sourceTag":"bug","summary":"issue tracker","url":"https://issues.chromium.org/issues/123456789"}],"severity":null,"summary":"Stable channel update rolling out to Windows, macOS, Linux.","title":"Stable Channel Update for Desktop"} \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html new file mode 100644 index 00000000..fa754870 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html @@ -0,0 +1,21 @@ + + + + + Stable Channel Update for Desktop + + +
    +

    The Stable channel has been updated to 128.0.6613.138 for Windows and macOS, and 128.0.6613.137 for Linux. A full list of changes in this build is available in the log.

    +

    The Extended Stable channel has been updated to 128.0.6613.138 for Windows and Mac and will roll out over the coming days.

    +

    The team is also rolling out Chrome 128.0.6613.89 to Android.

    +

    Security Fixes and Rewards

    +

    We would like to thank all security researchers who worked with us during the development cycle.

    +
      +
    • CVE-2024-12345: Use after free in Blink.
      • +
    • CVE-2024-22222: Heap buffer overflow in GPU.
      • +
    +

    For details see the issue tracker and the security page.

    +
    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml new file mode 100644 index 00000000..26357b2b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml @@ -0,0 +1,16 @@ + + + tag:blogger.com,1999:blog-8982037438137564684 + 2024-09-10T18:00:00Z + Google Chrome Releases + + + tag:blogger.com,1999:blog-8982037438137564684.post-123456789 + 2024-09-10T17:30:00Z + 2024-09-10T17:45:00Z + Stable Channel Update for Desktop + Stable channel update rolling out to Windows, macOS, Linux. + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj new file mode 100644 index 00000000..dd06a3fa --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md new file mode 100644 index 00000000..62ed5040 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Chromium/Chrome vendor feed connector parsing Stable Channel Update posts; authoritative vendor context for Chrome/Chromium versions and CVE lists; maps fixed versions as affected ranges. +## Scope +- Crawl Chrome Releases blog list; window by publish date; fetch detail posts; identify "Stable Channel Update" and security fix sections. +- Validate HTML; extract version trains, platform notes (Windows/macOS/Linux/Android), CVEs, acknowledgements; map fixed versions. +- Persist raw docs and maintain source_state cursor; idempotent mapping. +## Participants +- Source.Common (HTTP, HTML helpers, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). +- Models (canonical; affected ranges by product/version). +- Core/WebService (jobs: source:chromium:fetch|parse|map). +- Merge engine (later) to respect vendor PSIRT precedence for Chrome. +## Interfaces & contracts +- Aliases: CHROMIUM-POST: plus CVE ids. +- Affected: Vendor=Google, Product=Chrome/Chromium (platform tags), Type=vendor; Versions indicate introduced? (often unknown) and fixed (for example 127.0.6533.88); tags mark platforms. +- References: advisory (post URL), release notes, bug links; kind set appropriately. +- Provenance: method=parser; value=post slug; recordedAt=fetch time. +## In/Out of scope +In: vendor advisory mapping, fixed version emission per platform, psirt_flags vendor context. +Out: OS distro packaging semantics; bug bounty details beyond references. +## Observability & security expectations +- Metrics: chromium.fetch.items, chromium.parse.fail, chromium.map.affected_count. +- Logs: post slugs, version extracted, platform coverage, timing; allowlist blog host. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Chromium.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs new file mode 100644 index 00000000..843f0dbb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs @@ -0,0 +1,364 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; +using StellaOps.Feedser.Source.Vndr.Chromium.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Plugin; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Vndr.Chromium; + +public sealed class ChromiumConnector : IFeedConnector +{ + private static readonly JsonSchema Schema = ChromiumSchemaProvider.Schema; + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly ChromiumFeedLoader _feedLoader; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IJsonSchemaValidator _schemaValidator; + private readonly ChromiumOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ChromiumDiagnostics _diagnostics; + private readonly ILogger _logger; + + public ChromiumConnector( + ChromiumFeedLoader feedLoader, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IPsirtFlagStore psirtFlagStore, + ISourceStateRepository stateRepository, + IJsonSchemaValidator schemaValidator, + IOptions options, + TimeProvider? timeProvider, + ChromiumDiagnostics diagnostics, + ILogger logger) + { + _feedLoader = feedLoader ?? throw new ArgumentNullException(nameof(feedLoader)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VndrChromiumConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var (windowStart, windowEnd) = CalculateWindow(cursor, now); + + IReadOnlyList feedEntries; + _diagnostics.FetchAttempt(); + try + { + feedEntries = await _feedLoader.LoadAsync(windowStart, windowEnd, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Chromium feed load failed {Start}-{End}", windowStart, windowEnd); + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var fetchCache = new Dictionary(cursor.FetchCache, StringComparer.Ordinal); + var touchedResources = new HashSet(StringComparer.Ordinal); + + var candidates = feedEntries + .Where(static entry => entry.IsSecurityUpdate()) + .OrderBy(static entry => entry.Published) + .ToArray(); + + if (candidates.Length == 0) + { + var untouched = cursor + .WithLastPublished(cursor.LastPublished ?? windowEnd) + .WithFetchCache(fetchCache); + await UpdateCursorAsync(untouched, cancellationToken).ConfigureAwait(false); + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var maxPublished = cursor.LastPublished; + + foreach (var entry in candidates) + { + try + { + var cacheKey = entry.DetailUri.ToString(); + touchedResources.Add(cacheKey); + + var metadata = ChromiumDocumentMetadata.CreateMetadata(entry.PostId, entry.Title, entry.Published, entry.Updated, entry.Summary); + var request = new SourceFetchRequest(ChromiumOptions.HttpClientName, SourceName, entry.DetailUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (cursor.TryGetFetchCache(cacheKey, out var cached) && string.Equals(cached.Sha256, result.Document.Sha256, StringComparison.OrdinalIgnoreCase)) + { + _diagnostics.FetchUnchanged(); + fetchCache[cacheKey] = new ChromiumFetchCacheEntry(result.Document.Sha256); + await _documentStore.UpdateStatusAsync(result.Document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + if (!maxPublished.HasValue || entry.Published > maxPublished) + { + maxPublished = entry.Published; + } + + continue; + } + + _diagnostics.FetchDocument(); + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + + if (!maxPublished.HasValue || entry.Published > maxPublished) + { + maxPublished = entry.Published; + } + + fetchCache[cacheKey] = new ChromiumFetchCacheEntry(result.Document.Sha256); + } + catch (Exception ex) + { + _logger.LogError(ex, "Chromium fetch failed for {Uri}", entry.DetailUri); + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + if (touchedResources.Count > 0) + { + var keysToRemove = fetchCache.Keys.Where(key => !touchedResources.Contains(key)).ToArray(); + foreach (var key in keysToRemove) + { + fetchCache.Remove(key); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(cursor.PendingMappings) + .WithLastPublished(maxPublished ?? cursor.LastPublished ?? windowEnd) + .WithFetchCache(fetchCache); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Chromium document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + ChromiumDto dto; + try + { + var metadata = ChromiumDocumentMetadata.FromDocument(document); + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = Encoding.UTF8.GetString(content); + dto = ChromiumParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Chromium parse failed for {Uri}", document.Uri); + _diagnostics.ParseFailure(); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + using var jsonDocument = JsonDocument.Parse(json); + try + { + _schemaValidator.Validate(jsonDocument, Schema, dto.PostId); + } + catch (StellaOps.Feedser.Source.Common.Json.JsonSchemaValidationException ex) + { + _logger.LogError(ex, "Chromium schema validation failed for {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var payload = BsonDocument.Parse(json); + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var validatedAt = _timeProvider.GetUtcNow(); + + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "chromium.post.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "chromium.post.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + _diagnostics.ParseSuccess(); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var json = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson }); + var dto = JsonSerializer.Deserialize(json, SerializerOptions); + if (dto is null) + { + _logger.LogWarning("Chromium DTO deserialization failed for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var recordedAt = _timeProvider.GetUtcNow(); + var (advisory, flag) = ChromiumMapper.Map(dto, SourceName, recordedAt); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + _diagnostics.MapSuccess(); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return ChromiumCursor.FromBsonDocument(record?.Cursor); + } + + private async Task UpdateCursorAsync(ChromiumCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private (DateTimeOffset start, DateTimeOffset end) CalculateWindow(ChromiumCursor cursor, DateTimeOffset now) + { + var lastPublished = cursor.LastPublished ?? now - _options.InitialBackfill; + var start = lastPublished - _options.WindowOverlap; + var backfill = now - _options.InitialBackfill; + if (start < backfill) + { + start = backfill; + } + + var end = now; + if (end <= start) + { + end = start.AddHours(1); + } + + return (start, end); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs new file mode 100644 index 00000000..a7e9b9a1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Chromium; + +public sealed class VndrChromiumConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-chromium"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs new file mode 100644 index 00000000..b1e6fa1b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs @@ -0,0 +1,69 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.Source.Vndr.Chromium; + +public sealed class ChromiumDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Feedser.Source.Vndr.Chromium"; + public const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _mapSuccess; + + public ChromiumDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter( + name: "chromium.fetch.attempts", + unit: "operations", + description: "Number of Chromium fetch operations executed."); + _fetchDocuments = _meter.CreateCounter( + name: "chromium.fetch.documents", + unit: "documents", + description: "Count of Chromium advisory documents fetched successfully."); + _fetchFailures = _meter.CreateCounter( + name: "chromium.fetch.failures", + unit: "operations", + description: "Count of Chromium fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "chromium.fetch.unchanged", + unit: "documents", + description: "Count of Chromium documents skipped due to unchanged content."); + _parseSuccess = _meter.CreateCounter( + name: "chromium.parse.success", + unit: "documents", + description: "Count of Chromium documents parsed successfully."); + _parseFailures = _meter.CreateCounter( + name: "chromium.parse.failures", + unit: "documents", + description: "Count of Chromium documents that failed to parse."); + _mapSuccess = _meter.CreateCounter( + name: "chromium.map.success", + unit: "advisories", + description: "Count of Chromium advisories mapped successfully."); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void MapSuccess() => _mapSuccess.Add(1); + + public Meter Meter => _meter; + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs new file mode 100644 index 00000000..4799bb30 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; +using StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +namespace StellaOps.Feedser.Source.Vndr.Chromium; + +public static class ChromiumServiceCollectionExtensions +{ + public static IServiceCollection AddChromiumConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSingleton(static sp => sp.GetRequiredService>().Value); + + services.AddSourceHttpClient(ChromiumOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = new Uri(options.FeedUri.GetLeftPart(UriPartial.Authority)); + clientOptions.Timeout = TimeSpan.FromSeconds(20); + clientOptions.UserAgent = "StellaOps.Feedser.VndrChromium/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.FeedUri.Host); + }); + + services.AddSingleton(); + services.AddTransient(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs new file mode 100644 index 00000000..7b113f0d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs @@ -0,0 +1,44 @@ +namespace StellaOps.Feedser.Source.Vndr.Chromium.Configuration; + +public sealed class ChromiumOptions +{ + public const string HttpClientName = "source-vndr-chromium"; + + public Uri FeedUri { get; set; } = new("https://chromereleases.googleblog.com/atom.xml"); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan WindowOverlap { get; set; } = TimeSpan.FromDays(2); + + public int MaxFeedPages { get; set; } = 4; + + public int MaxEntriesPerPage { get; set; } = 50; + + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new ArgumentException("FeedUri must be an absolute URI.", nameof(FeedUri)); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new ArgumentException("InitialBackfill must be positive.", nameof(InitialBackfill)); + } + + if (WindowOverlap < TimeSpan.Zero) + { + throw new ArgumentException("WindowOverlap cannot be negative.", nameof(WindowOverlap)); + } + + if (MaxFeedPages <= 0) + { + throw new ArgumentException("MaxFeedPages must be positive.", nameof(MaxFeedPages)); + } + + if (MaxEntriesPerPage <= 0 || MaxEntriesPerPage > 100) + { + throw new ArgumentException("MaxEntriesPerPage must be between 1 and 100.", nameof(MaxEntriesPerPage)); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs new file mode 100644 index 00000000..3ce2c3c0 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs @@ -0,0 +1,143 @@ +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal sealed record ChromiumCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary FetchCache) +{ + public static ChromiumCursor Empty { get; } = new(null, Array.Empty(), Array.Empty(), new Dictionary(StringComparer.Ordinal)); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())); + + if (FetchCache.Count > 0) + { + var cacheDocument = new BsonDocument(); + foreach (var (key, entry) in FetchCache) + { + cacheDocument[key] = entry.ToBson(); + } + + document["fetchCache"] = cacheDocument; + } + + return document; + } + + public static ChromiumCursor FromBsonDocument(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + DateTimeOffset? lastPublished = null; + if (document.TryGetValue("lastPublished", out var lastPublishedValue)) + { + lastPublished = ReadDateTime(lastPublishedValue); + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var fetchCache = ReadFetchCache(document); + + return new ChromiumCursor(lastPublished, pendingDocuments, pendingMappings, fetchCache); + } + + public ChromiumCursor WithLastPublished(DateTimeOffset? lastPublished) + => this with { LastPublished = lastPublished?.ToUniversalTime() }; + + public ChromiumCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public ChromiumCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public ChromiumCursor WithFetchCache(IDictionary cache) + => this with { FetchCache = cache is null ? new Dictionary(StringComparer.Ordinal) : new Dictionary(cache, StringComparer.Ordinal) }; + + public bool TryGetFetchCache(string key, out ChromiumFetchCacheEntry entry) + => FetchCache.TryGetValue(key, out entry); + + private static DateTimeOffset? ReadDateTime(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static IReadOnlyDictionary ReadFetchCache(BsonDocument document) + { + if (!document.TryGetValue("fetchCache", out var value) || value is not BsonDocument cacheDocument) + { + return new Dictionary(StringComparer.Ordinal); + } + + var dictionary = new Dictionary(StringComparer.Ordinal); + foreach (var element in cacheDocument.Elements) + { + if (element.Value is BsonDocument entryDocument) + { + dictionary[element.Name] = ChromiumFetchCacheEntry.FromBson(entryDocument); + } + } + + return dictionary; + } +} + +internal sealed record ChromiumFetchCacheEntry(string Sha256) +{ + public static ChromiumFetchCacheEntry Empty { get; } = new(string.Empty); + + public BsonDocument ToBson() + { + var document = new BsonDocument + { + ["sha256"] = Sha256, + }; + + return document; + } + + public static ChromiumFetchCacheEntry FromBson(BsonDocument document) + { + var sha = document.TryGetValue("sha256", out var shaValue) ? shaValue.AsString : string.Empty; + return new ChromiumFetchCacheEntry(sha); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs new file mode 100644 index 00000000..0ab4ef77 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs @@ -0,0 +1,78 @@ +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal sealed record ChromiumDocumentMetadata( + string PostId, + string Title, + Uri DetailUrl, + DateTimeOffset Published, + DateTimeOffset? Updated, + string? Summary) +{ + private const string PostIdKey = "postId"; + private const string TitleKey = "title"; + private const string PublishedKey = "published"; + private const string UpdatedKey = "updated"; + private const string SummaryKey = "summary"; + + public static ChromiumDocumentMetadata FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + + var metadata = document.Metadata ?? throw new InvalidOperationException("Chromium document metadata missing."); + + if (!metadata.TryGetValue(PostIdKey, out var postId) || string.IsNullOrWhiteSpace(postId)) + { + throw new InvalidOperationException("Chromium document metadata missing postId."); + } + + if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title)) + { + throw new InvalidOperationException("Chromium document metadata missing title."); + } + + if (!metadata.TryGetValue(PublishedKey, out var publishedString) || !DateTimeOffset.TryParse(publishedString, out var published)) + { + throw new InvalidOperationException("Chromium document metadata missing published timestamp."); + } + + DateTimeOffset? updated = null; + if (metadata.TryGetValue(UpdatedKey, out var updatedString) && DateTimeOffset.TryParse(updatedString, out var updatedValue)) + { + updated = updatedValue; + } + + metadata.TryGetValue(SummaryKey, out var summary); + + return new ChromiumDocumentMetadata( + postId.Trim(), + title.Trim(), + new Uri(document.Uri, UriKind.Absolute), + published.ToUniversalTime(), + updated?.ToUniversalTime(), + string.IsNullOrWhiteSpace(summary) ? null : summary.Trim()); + } + + public static IReadOnlyDictionary CreateMetadata(string postId, string title, DateTimeOffset published, DateTimeOffset? updated, string? summary) + { + var dictionary = new Dictionary(StringComparer.Ordinal) + { + [PostIdKey] = postId, + [TitleKey] = title, + [PublishedKey] = published.ToUniversalTime().ToString("O"), + }; + + if (updated.HasValue) + { + dictionary[UpdatedKey] = updated.Value.ToUniversalTime().ToString("O"); + } + + if (!string.IsNullOrWhiteSpace(summary)) + { + dictionary[SummaryKey] = summary.Trim(); + } + + return dictionary; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs new file mode 100644 index 00000000..5a33dbc1 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs @@ -0,0 +1,39 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal sealed record ChromiumDto( + [property: JsonPropertyName("postId")] string PostId, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("detailUrl")] string DetailUrl, + [property: JsonPropertyName("published")] DateTimeOffset Published, + [property: JsonPropertyName("updated")] DateTimeOffset? Updated, + [property: JsonPropertyName("summary")] string? Summary, + [property: JsonPropertyName("cves")] IReadOnlyList Cves, + [property: JsonPropertyName("platforms")] IReadOnlyList Platforms, + [property: JsonPropertyName("versions")] IReadOnlyList Versions, + [property: JsonPropertyName("references")] IReadOnlyList References) +{ + public static ChromiumDto From(ChromiumDocumentMetadata metadata, IReadOnlyList cves, IReadOnlyList platforms, IReadOnlyList versions, IReadOnlyList references) + => new( + metadata.PostId, + metadata.Title, + metadata.DetailUrl.ToString(), + metadata.Published, + metadata.Updated, + metadata.Summary, + cves, + platforms, + versions, + references); +} + +internal sealed record ChromiumVersionInfo( + [property: JsonPropertyName("platform")] string Platform, + [property: JsonPropertyName("channel")] string Channel, + [property: JsonPropertyName("version")] string Version); + +internal sealed record ChromiumReference( + [property: JsonPropertyName("url")] string Url, + [property: JsonPropertyName("kind")] string Kind, + [property: JsonPropertyName("label")] string? Label); diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs new file mode 100644 index 00000000..39e8e46f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs @@ -0,0 +1,24 @@ +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +public sealed record ChromiumFeedEntry( + string EntryId, + string PostId, + string Title, + Uri DetailUri, + DateTimeOffset Published, + DateTimeOffset? Updated, + string? Summary, + IReadOnlyCollection Categories) +{ + public bool IsSecurityUpdate() + { + if (Categories.Count > 0 && Categories.Contains("Stable updates", StringComparer.OrdinalIgnoreCase)) + { + return true; + } + + return Title.Contains("Stable Channel Update", StringComparison.OrdinalIgnoreCase) + || Title.Contains("Extended Stable", StringComparison.OrdinalIgnoreCase) + || Title.Contains("Stable Channel Desktop", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs new file mode 100644 index 00000000..ee3fd738 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs @@ -0,0 +1,147 @@ +using System.ServiceModel.Syndication; +using System.Xml; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +public sealed class ChromiumFeedLoader +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly ChromiumOptions _options; + private readonly ILogger _logger; + + public ChromiumFeedLoader(IHttpClientFactory httpClientFactory, IOptions options, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> LoadAsync(DateTimeOffset windowStart, DateTimeOffset windowEnd, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(ChromiumOptions.HttpClientName); + var results = new List(); + var startIndex = 1; + + for (var page = 0; page < _options.MaxFeedPages; page++) + { + var requestUri = BuildRequestUri(startIndex); + using var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var reader = XmlReader.Create(stream); + var feed = SyndicationFeed.Load(reader); + if (feed is null || feed.Items is null) + { + break; + } + + var pageEntries = new List(); + foreach (var entry in feed.Items) + { + var published = entry.PublishDate != DateTimeOffset.MinValue + ? entry.PublishDate.ToUniversalTime() + : entry.LastUpdatedTime.ToUniversalTime(); + + if (published > windowEnd || published < windowStart - _options.WindowOverlap) + { + continue; + } + + var detailUri = entry.Links.FirstOrDefault(link => string.Equals(link.RelationshipType, "alternate", StringComparison.OrdinalIgnoreCase))?.Uri; + if (detailUri is null) + { + continue; + } + + var postId = ExtractPostId(detailUri); + if (string.IsNullOrEmpty(postId)) + { + continue; + } + + var categories = entry.Categories.Select(static cat => cat.Name).Where(static name => !string.IsNullOrWhiteSpace(name)).ToArray(); + var chromiumEntry = new ChromiumFeedEntry( + entry.Id ?? detailUri.ToString(), + postId, + entry.Title?.Text?.Trim() ?? postId, + detailUri, + published, + entry.LastUpdatedTime == DateTimeOffset.MinValue ? null : entry.LastUpdatedTime.ToUniversalTime(), + entry.Summary?.Text?.Trim(), + categories); + + if (chromiumEntry.Published >= windowStart && chromiumEntry.Published <= windowEnd) + { + pageEntries.Add(chromiumEntry); + } + } + + if (pageEntries.Count == 0) + { + var oldest = feed.Items?.Select(static item => item.PublishDate).Where(static dt => dt != DateTimeOffset.MinValue).OrderBy(static dt => dt).FirstOrDefault(); + if (oldest.HasValue && oldest.Value.ToUniversalTime() < windowStart) + { + break; + } + } + + results.AddRange(pageEntries); + + if (feed.Items?.Any() != true) + { + break; + } + + var nextLink = feed.Links?.FirstOrDefault(link => string.Equals(link.RelationshipType, "next", StringComparison.OrdinalIgnoreCase))?.Uri; + if (nextLink is null) + { + break; + } + + startIndex += _options.MaxEntriesPerPage; + } + + return results + .DistinctBy(static entry => entry.DetailUri) + .OrderBy(static entry => entry.Published) + .ToArray(); + } + + private Uri BuildRequestUri(int startIndex) + { + var builder = new UriBuilder(_options.FeedUri); + var query = new List(); + + if (!string.IsNullOrEmpty(builder.Query)) + { + query.Add(builder.Query.TrimStart('?')); + } + + query.Add($"max-results={_options.MaxEntriesPerPage}"); + query.Add($"start-index={startIndex}"); + query.Add("redirect=false"); + builder.Query = string.Join('&', query); + return builder.Uri; + } + + private static string ExtractPostId(Uri detailUri) + { + var segments = detailUri.Segments; + if (segments.Length == 0) + { + return detailUri.AbsoluteUri; + } + + var last = segments[^1].Trim('/'); + if (last.EndsWith(".html", StringComparison.OrdinalIgnoreCase)) + { + last = last[..^5]; + } + + return last.Replace('/', '-'); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs new file mode 100644 index 00000000..957b58fa --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs @@ -0,0 +1,120 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal static class ChromiumMapper +{ + private const string VendorIdentifier = "google:chrome"; + + public static (Advisory Advisory, PsirtFlagRecord Flag) Map(ChromiumDto dto, string sourceName, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + var advisoryKey = $"chromium/post/{dto.PostId}"; + var provenance = new AdvisoryProvenance(sourceName, "document", dto.PostId, recordedAt.ToUniversalTime()); + + var aliases = BuildAliases(dto).ToArray(); + var references = BuildReferences(dto, provenance).ToArray(); + var affectedPackages = BuildAffected(dto, provenance).ToArray(); + + var advisory = new Advisory( + advisoryKey, + dto.Title, + dto.Summary, + language: "en", + dto.Published.ToUniversalTime(), + dto.Updated?.ToUniversalTime(), + severity: null, + exploitKnown: false, + aliases, + references, + affectedPackages, + Array.Empty(), + new[] { provenance }); + + var flag = new PsirtFlagRecord( + advisoryKey, + "Google", + sourceName, + dto.PostId, + recordedAt.ToUniversalTime()); + + return (advisory, flag); + } + + private static IEnumerable BuildAliases(ChromiumDto dto) + { + yield return $"CHROMIUM-POST:{dto.PostId}"; + yield return $"CHROMIUM-POST:{dto.Published:yyyy-MM-dd}"; + + foreach (var cve in dto.Cves) + { + yield return cve; + } + } + + private static IEnumerable BuildReferences(ChromiumDto dto, AdvisoryProvenance provenance) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var references = new List<(AdvisoryReference Reference, int Priority)> + { + (new AdvisoryReference(dto.DetailUrl, "advisory", "chromium-blog", summary: null, provenance), 0), + }; + + foreach (var reference in dto.References) + { + var summary = string.IsNullOrWhiteSpace(reference.Label) ? null : reference.Label; + var sourceTag = string.IsNullOrWhiteSpace(reference.Kind) ? null : reference.Kind; + var advisoryReference = new AdvisoryReference(reference.Url, reference.Kind, sourceTag, summary, provenance); + references.Add((advisoryReference, 1)); + } + + return references + .GroupBy(tuple => tuple.Reference.Url, comparer) + .Select(group => group + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.SourceTag ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .First()) + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .Select(t => t.Reference); + } + + private static IEnumerable BuildAffected(ChromiumDto dto, AdvisoryProvenance provenance) + { + foreach (var version in dto.Versions) + { + var identifier = version.Channel switch + { + "extended-stable" => $"{VendorIdentifier}:extended-stable", + "beta" => $"{VendorIdentifier}:beta", + "dev" => $"{VendorIdentifier}:dev", + _ => VendorIdentifier, + }; + + var range = new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: version.Version, + lastAffectedVersion: null, + rangeExpression: null, + provenance); + + yield return new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + version.Platform, + new[] { range }, + statuses: Array.Empty(), + provenance: new[] { provenance }); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs new file mode 100644 index 00000000..2febc601 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs @@ -0,0 +1,282 @@ +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal static class ChromiumParser +{ + private static readonly HtmlParser HtmlParser = new(); + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,}", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex VersionRegex = new("(?\\d+\\.\\d+\\.\\d+\\.\\d+)", RegexOptions.Compiled); + + public static ChromiumDto Parse(string html, ChromiumDocumentMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + var document = HtmlParser.ParseDocument(html); + var body = document.QuerySelector("div.post-body") ?? document.Body; + if (body is null) + { + throw new InvalidOperationException("Chromium post body not found."); + } + + var cves = ExtractCves(body); + var versions = ExtractVersions(body); + var platforms = versions.Select(static v => v.Platform).Distinct(StringComparer.OrdinalIgnoreCase).ToArray(); + var references = ExtractReferences(body, metadata.DetailUrl); + + return ChromiumDto.From(metadata, cves, platforms, versions, references); + } + + private static IReadOnlyList ExtractCves(IElement body) + { + var matches = CveRegex.Matches(body.TextContent ?? string.Empty); + return matches + .Select(static match => match.Value.ToUpperInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static cve => cve, StringComparer.Ordinal) + .ToArray(); + } + + private static IReadOnlyList ExtractVersions(IElement body) + { + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + var elements = body.QuerySelectorAll("p,li"); + if (elements.Length == 0) + { + elements = body.QuerySelectorAll("div,span"); + } + + foreach (var element in elements) + { + var text = element.TextContent?.Trim(); + if (string.IsNullOrEmpty(text)) + { + continue; + } + + var channel = DetermineChannel(text); + foreach (Match match in VersionRegex.Matches(text)) + { + var version = match.Groups["version"].Value; + var platform = DeterminePlatform(text, match); + var key = string.Join('|', platform.ToLowerInvariant(), channel.ToLowerInvariant(), version); + if (!results.ContainsKey(key)) + { + results[key] = new ChromiumVersionInfo(platform, channel, version); + } + } + } + + return results.Values + .OrderBy(static v => v.Platform, StringComparer.OrdinalIgnoreCase) + .ThenBy(static v => v.Channel, StringComparer.OrdinalIgnoreCase) + .ThenBy(static v => v.Version, StringComparer.Ordinal) + .ToArray(); + } + + private static string DeterminePlatform(string text, Match match) + { + var after = ExtractSlice(text, match.Index + match.Length, Math.Min(120, text.Length - (match.Index + match.Length))); + var segment = ExtractPlatformSegment(after); + var normalized = NormalizePlatform(segment); + if (!string.IsNullOrEmpty(normalized)) + { + return normalized!; + } + + var before = ExtractSlice(text, Math.Max(0, match.Index - 80), Math.Min(80, match.Index)); + normalized = NormalizePlatform(before + " " + after); + return string.IsNullOrEmpty(normalized) ? "desktop" : normalized!; + } + + private static string DetermineChannel(string text) + { + if (text.Contains("Extended Stable", StringComparison.OrdinalIgnoreCase)) + { + return "extended-stable"; + } + + if (text.Contains("Beta", StringComparison.OrdinalIgnoreCase)) + { + return "beta"; + } + + if (text.Contains("Dev", StringComparison.OrdinalIgnoreCase)) + { + return "dev"; + } + + return "stable"; + } + + private static string ExtractSlice(string text, int start, int length) + { + if (length <= 0) + { + return string.Empty; + } + + return text.Substring(start, length); + } + + private static string ExtractPlatformSegment(string after) + { + if (string.IsNullOrEmpty(after)) + { + return string.Empty; + } + + var forIndex = after.IndexOf("for ", StringComparison.OrdinalIgnoreCase); + if (forIndex < 0) + { + return string.Empty; + } + + var remainder = after[(forIndex + 4)..]; + var terminatorIndex = remainder.IndexOfAny(new[] { '.', ';', '\n', '(', ')' }); + if (terminatorIndex >= 0) + { + remainder = remainder[..terminatorIndex]; + } + + var digitIndex = remainder.IndexOfAny("0123456789".ToCharArray()); + if (digitIndex >= 0) + { + remainder = remainder[..digitIndex]; + } + + var whichIndex = remainder.IndexOf(" which", StringComparison.OrdinalIgnoreCase); + if (whichIndex >= 0) + { + remainder = remainder[..whichIndex]; + } + + return remainder.Trim(); + } + + private static string? NormalizePlatform(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var normalized = value.Replace("/", " ", StringComparison.OrdinalIgnoreCase) + .Replace(" and ", " ", StringComparison.OrdinalIgnoreCase) + .Replace("&", " ", StringComparison.OrdinalIgnoreCase) + .Trim(); + + if (normalized.Contains("android", StringComparison.OrdinalIgnoreCase)) + { + return "android"; + } + + if (normalized.Contains("chromeos flex", StringComparison.OrdinalIgnoreCase)) + { + return "chromeos-flex"; + } + + if (normalized.Contains("chromeos", StringComparison.OrdinalIgnoreCase) || normalized.Contains("chrome os", StringComparison.OrdinalIgnoreCase)) + { + return "chromeos"; + } + + if (normalized.Contains("linux", StringComparison.OrdinalIgnoreCase)) + { + return "linux"; + } + + var hasWindows = normalized.Contains("windows", StringComparison.OrdinalIgnoreCase); + var hasMac = normalized.Contains("mac", StringComparison.OrdinalIgnoreCase); + + if (hasWindows && hasMac) + { + return "windows-mac"; + } + + if (hasWindows) + { + return "windows"; + } + + if (hasMac) + { + return "mac"; + } + + return null; + } + + private static IReadOnlyList ExtractReferences(IElement body, Uri detailUri) + { + var references = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var anchor in body.QuerySelectorAll("a[href]")) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!Uri.TryCreate(href.Trim(), UriKind.Absolute, out var linkUri)) + { + continue; + } + + if (string.Equals(linkUri.AbsoluteUri, detailUri.AbsoluteUri, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!string.Equals(linkUri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(linkUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var kind = ClassifyReference(linkUri); + var label = anchor.TextContent?.Trim(); + + if (!references.ContainsKey(linkUri.AbsoluteUri)) + { + references[linkUri.AbsoluteUri] = new ChromiumReference(linkUri.AbsoluteUri, kind, string.IsNullOrWhiteSpace(label) ? null : label); + } + } + + return references.Values + .OrderBy(static r => r.Url, StringComparer.Ordinal) + .ThenBy(static r => r.Kind, StringComparer.Ordinal) + .ToArray(); + } + + private static string ClassifyReference(Uri uri) + { + var host = uri.Host; + if (host.Contains("googlesource.com", StringComparison.OrdinalIgnoreCase)) + { + return "changelog"; + } + + if (host.Contains("issues.chromium.org", StringComparison.OrdinalIgnoreCase) + || host.Contains("bugs.chromium.org", StringComparison.OrdinalIgnoreCase) + || host.Contains("crbug.com", StringComparison.OrdinalIgnoreCase)) + { + return "bug"; + } + + if (host.Contains("chromium.org", StringComparison.OrdinalIgnoreCase)) + { + return "doc"; + } + + if (host.Contains("google.com", StringComparison.OrdinalIgnoreCase)) + { + return "google"; + } + + return "reference"; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs new file mode 100644 index 00000000..9cccf4d5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs @@ -0,0 +1,25 @@ +using System.IO; +using System.Reflection; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; + +internal static class ChromiumSchemaProvider +{ + private static readonly Lazy Cached = new(Load, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => Cached.Value; + + private static JsonSchema Load() + { + var assembly = typeof(ChromiumSchemaProvider).GetTypeInfo().Assembly; + const string resourceName = "StellaOps.Feedser.Source.Vndr.Chromium.Schemas.chromium-post.schema.json"; + + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); + using var reader = new StreamReader(stream); + var schemaText = reader.ReadToEnd(); + return JsonSchema.FromText(schemaText); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..c682035d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Chromium.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json b/src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json new file mode 100644 index 00000000..8dc8b547 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://stellaops.example/schemas/chromium-post.schema.json", + "type": "object", + "required": [ + "postId", + "title", + "detailUrl", + "published", + "cves", + "platforms", + "versions", + "references" + ], + "properties": { + "postId": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string", + "minLength": 1 + }, + "detailUrl": { + "type": "string", + "format": "uri" + }, + "published": { + "type": "string", + "format": "date-time" + }, + "updated": { + "type": ["string", "null"], + "format": "date-time" + }, + "summary": { + "type": ["string", "null"] + }, + "cves": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^CVE-\\d{4}-\\d{4,}$" + } + }, + "platforms": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "versions": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["platform", "channel", "version"], + "properties": { + "platform": { + "type": "string", + "minLength": 1 + }, + "channel": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 4 + } + } + } + }, + "references": { + "type": "array", + "items": { + "type": "object", + "required": ["url", "kind"], + "properties": { + "url": { + "type": "string", + "format": "uri" + }, + "kind": { + "type": "string", + "minLength": 1 + }, + "label": { + "type": ["string", "null"] + } + } + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj b/src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj new file mode 100644 index 00000000..fca938f3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj @@ -0,0 +1,32 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + + + + + + + <_Parameter1>StellaOps.Feedser.Source.Vndr.Chromium.Tests + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md new file mode 100644 index 00000000..b560dfbb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md @@ -0,0 +1,16 @@ +# Source.Vndr.Chromium — Task Board + +| ID | Task | Owner | Status | Depends On | Notes | +|------|-----------------------------------------------|-------|--------|------------|-------| +| CH1 | Blog crawl + cursor | Conn | DONE | Common | Sliding window feed reader with cursor persisted. | +| CH2 | Post parser → DTO (CVEs, versions, refs) | QA | DONE | | AngleSharp parser normalizes CVEs, versions, references. | +| CH3 | Canonical mapping (aliases/refs/affected-hint)| Conn | DONE | Models | Deterministic advisory mapping with psirt flags. | +| CH4 | Snapshot tests + resume | QA | DONE | Storage | Deterministic snapshot plus resume scenario via Mongo state. | +| CH5 | Observability | QA | DONE | | Metered fetch/parse/map counters. | +| CH6 | SourceState + SHA dedupe | Conn | DONE | Storage | Cursor tracks SHA cache to skip unchanged posts. | +| CH7 | Stabilize resume integration (preserve pending docs across provider instances) | QA | DONE | Storage.Mongo | Resume integration test exercises pending docs across providers via shared Mongo. | +| CH8 | Mark failed parse documents | Conn | DONE | Storage.Mongo | Parse pipeline marks failures; unit tests assert status transitions. | +| CH9 | Reference dedupe & ordering | Conn | DONE | Models | Mapper groups references by URL and sorts deterministically. | + +## Changelog +- YYYY-MM-DD: Created. diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Class1.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Class1.cs new file mode 100644 index 00000000..e527bff6 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Cisco/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Cisco; + +public sealed class VndrCiscoConnectorPlugin : IConnectorPlugin +{ + public string Name => "vndr-cisco"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj b/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Class1.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Class1.cs new file mode 100644 index 00000000..d0034d98 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Msrc/Class1.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Msrc; + +public sealed class VndrMsrcConnectorPlugin : IConnectorPlugin +{ + public string Name => "vndr-msrc"; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name); + + private sealed class StubConnector : IFeedConnector + { + public StubConnector(string sourceName) => SourceName = sourceName; + + public string SourceName { get; } + + public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask; + } +} + diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj b/src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj new file mode 100644 index 00000000..182529d4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json new file mode 100644 index 00000000..9d60f230 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json @@ -0,0 +1,112 @@ +[ + { + "advisoryKey": "oracle/cpuapr2024-01-html", + "affectedPackages": [], + "aliases": [ + "ORACLE:cpuapr2024-01-html" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + } + ], + "published": "2024-04-18T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "kind": "document", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.oracle.com/kb/123456" + }, + { + "kind": "reference", + "provenance": { + "kind": "document", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + }, + "sourceTag": null, + "summary": null, + "url": "https://updates.oracle.com/patches/patch01" + }, + { + "kind": "advisory", + "provenance": { + "kind": "document", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + }, + "sourceTag": "oracle", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpuapr2024-01.html" + } + ], + "severity": null, + "summary": "Oracle CPU April 2024 Advisory 1 Oracle Critical Patch Update Advisory - April 2024 (CPU01) This advisory addresses vulnerabilities in Oracle Database Server. Patch download Support article", + "title": "cpuapr2024 01 html" + }, + { + "advisoryKey": "oracle/cpuapr2024-02-html", + "affectedPackages": [], + "aliases": [ + "ORACLE:cpuapr2024-02-html" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "kind": "document", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + } + ], + "published": "2024-04-18T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "kind": "document", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.oracle.com/kb/789012" + }, + { + "kind": "advisory", + "provenance": { + "kind": "document", + "recordedAt": "2024-04-18T00:01:00+00:00", + "source": "vndr-oracle", + "value": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + }, + "sourceTag": "oracle", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpuapr2024-02.html" + } + ], + "severity": null, + "summary": "Oracle CPU April 2024 Advisory 2 Oracle Security Alert Advisory - April 2024 (CPU02) Mitigations for Oracle WebLogic Server. More details at Support KB .", + "title": "cpuapr2024 02 html" + } +] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html new file mode 100644 index 00000000..7ffd86f7 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html @@ -0,0 +1,11 @@ + + Oracle CPU April 2024 Advisory 1 + +

    Oracle Critical Patch Update Advisory - April 2024 (CPU01)

    +

    This advisory addresses vulnerabilities in Oracle Database Server.

    + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html new file mode 100644 index 00000000..764eaaa8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html @@ -0,0 +1,8 @@ + + Oracle CPU April 2024 Advisory 2 + +

    Oracle Security Alert Advisory - April 2024 (CPU02)

    +

    Mitigations for Oracle WebLogic Server.

    +

    More details at Support KB.

    + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs new file mode 100644 index 00000000..5d38b5b3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Source.Vndr.Oracle; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Tests; + +[Collection("mongo-fixture")] +public sealed class OracleConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + private static readonly Uri AdvisoryOne = new("https://www.oracle.com/security-alerts/cpuapr2024-01.html"); + private static readonly Uri AdvisoryTwo = new("https://www.oracle.com/security-alerts/cpuapr2024-02.html"); + + public OracleConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 4, 18, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_EmitsOraclePsirtSnapshot() + { + await using var provider = await BuildServiceProviderAsync(); + SeedDetails(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var snapshot = SnapshotSerializer.ToSnapshot(advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray()); + var expected = ReadFixture("oracle-advisories.snapshot.json"); + var normalizedSnapshot = Normalize(snapshot); + var normalizedExpected = Normalize(expected); + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Oracle", "Fixtures", "oracle-advisories.actual.json"); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + + var psirtCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + var flags = await psirtCollection.Find(Builders.Filter.Empty).ToListAsync(); + Assert.Equal(2, flags.Count); + Assert.All(flags, doc => Assert.Equal("Oracle", doc["vendor"].AsString)); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddOracleConnector(opts => + { + opts.AdvisoryUris = new List { AdvisoryOne, AdvisoryTwo }; + opts.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(OracleOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedDetails() + { + AddDetailResponse(AdvisoryOne, "oracle-detail-cpuapr2024-01.html", "\"oracle-001\""); + AddDetailResponse(AdvisoryTwo, "oracle-detail-cpuapr2024-02.html", "\"oracle-002\""); + } + + private void AddDetailResponse(Uri uri, string fixture, string? etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + + if (!string.IsNullOrEmpty(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static string ReadFixture(string filename) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Oracle", "Fixtures", filename); + return File.ReadAllText(path); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj new file mode 100644 index 00000000..b9a6a2ee --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md new file mode 100644 index 00000000..8130f592 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +Oracle PSIRT connector for Critical Patch Updates (CPU) and Security Alerts; authoritative vendor ranges and severities for Oracle products; establishes PSIRT precedence over registry or distro where applicable. +## Scope +- Harvest CPU calendar pages and per-advisory content; window by CPU cycle (Jan/Apr/Jul/Oct) and last modified timestamps. +- Validate HTML or JSON; extract CVE lists, affected products, components, versions, fixed patch levels; map to canonical with aliases and psirt_flags. +- Persist raw documents; maintain source_state across cycles; idempotent mapping. +## Participants +- Source.Common (HTTP, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). +- Models (canonical; affected ranges for vendor products). +- Core/WebService (jobs: source:oracle:fetch|parse|map). +- Merge engine (later) to prefer PSIRT ranges over NVD for Oracle products. +## Interfaces & contracts +- Alias scheme includes CPU:YYYY-QQ plus individual advisory ids when present; include CVE mappings. +- Affected entries capture product/component and fixedBy patch version; references include product notes and patch docs; kind=advisory or patch. +- Provenance.method=parser; value includes CPU cycle and advisory slug. +## In/Out of scope +In: PSIRT authoritative mapping, cycles handling, precedence signaling. +Out: signing or patch artifact downloads. +## Observability & security expectations +- Metrics: oracle.fetch.pages, oracle.cpu.cycles, oracle.parse.fail, oracle.map.affected_count. +- Logs: cycle tags, advisory ids, extraction timings; redact nothing sensitive. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Oracle.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs new file mode 100644 index 00000000..32d1ca6c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Configuration; + +public sealed class OracleOptions +{ + public const string HttpClientName = "vndr-oracle"; + + public List AdvisoryUris { get; set; } = new(); + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromSeconds(1); + + public void Validate() + { + if (AdvisoryUris.Count == 0) + { + throw new InvalidOperationException("Oracle AdvisoryUris must include at least one URI."); + } + + if (AdvisoryUris.Any(uri => uri is null || !uri.IsAbsoluteUri)) + { + throw new InvalidOperationException("All Oracle AdvisoryUris must be absolute URIs."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs new file mode 100644 index 00000000..f203a487 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OracleCursor( + DateTimeOffset? LastProcessed, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static OracleCursor Empty { get; } = new(null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastProcessed.HasValue) + { + document["lastProcessed"] = LastProcessed.Value.UtcDateTime; + } + + return document; + } + + public static OracleCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastProcessed = document.TryGetValue("lastProcessed", out var value) + ? ParseDate(value) + : null; + + return new OracleCursor( + lastProcessed, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings")); + } + + public OracleCursor WithLastProcessed(DateTimeOffset? timestamp) + => this with { LastProcessed = timestamp }; + + public OracleCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public OracleCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var raw) || raw is not BsonArray array) + { + return Array.Empty(); + } + + var result = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + result.Add(guid); + } + } + + return result; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs new file mode 100644 index 00000000..3cb123a8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OracleDocumentMetadata( + string AdvisoryId, + string Title, + DateTimeOffset Published, + Uri DetailUri) +{ + private const string AdvisoryIdKey = "oracle.advisoryId"; + private const string TitleKey = "oracle.title"; + private const string PublishedKey = "oracle.published"; + + public static IReadOnlyDictionary CreateMetadata(string advisoryId, string title, DateTimeOffset published) + => new Dictionary(StringComparer.Ordinal) + { + [AdvisoryIdKey] = advisoryId, + [TitleKey] = title, + [PublishedKey] = published.ToString("O"), + }; + + public static OracleDocumentMetadata FromDocument(DocumentRecord document) + { + ArgumentNullException.ThrowIfNull(document); + if (document.Metadata is null) + { + throw new InvalidOperationException("Oracle document metadata missing."); + } + + var metadata = document.Metadata; + if (!metadata.TryGetValue(AdvisoryIdKey, out var advisoryId) || string.IsNullOrWhiteSpace(advisoryId)) + { + throw new InvalidOperationException("Oracle advisory id metadata missing."); + } + + if (!metadata.TryGetValue(TitleKey, out var title) || string.IsNullOrWhiteSpace(title)) + { + throw new InvalidOperationException("Oracle title metadata missing."); + } + + if (!metadata.TryGetValue(PublishedKey, out var publishedRaw) || !DateTimeOffset.TryParse(publishedRaw, out var published)) + { + throw new InvalidOperationException("Oracle published metadata invalid."); + } + + if (!Uri.TryCreate(document.Uri, UriKind.Absolute, out var detailUri)) + { + throw new InvalidOperationException("Oracle document URI invalid."); + } + + return new OracleDocumentMetadata(advisoryId.Trim(), title.Trim(), published.ToUniversalTime(), detailUri); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs new file mode 100644 index 00000000..a273d71d --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal sealed record OracleDto( + [property: JsonPropertyName("advisoryId")] string AdvisoryId, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("detailUrl")] string DetailUrl, + [property: JsonPropertyName("published")] DateTimeOffset Published, + [property: JsonPropertyName("content")] string Content, + [property: JsonPropertyName("references")] IReadOnlyList References); diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs new file mode 100644 index 00000000..79f3c5b5 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal static class OracleMapper +{ + public static (Advisory Advisory, PsirtFlagRecord Flag) Map(OracleDto dto, string sourceName, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + var advisoryKey = $"oracle/{dto.AdvisoryId}"; + var provenance = new AdvisoryProvenance(sourceName, "document", dto.DetailUrl, recordedAt.ToUniversalTime()); + + var aliases = new List + { + $"ORACLE:{dto.AdvisoryId}", + }; + + var references = BuildReferences(dto, provenance).ToArray(); + + var advisory = new Advisory( + advisoryKey, + dto.Title, + dto.Content, + language: "en", + published: dto.Published.ToUniversalTime(), + modified: null, + severity: null, + exploitKnown: false, + aliases, + references, + Array.Empty(), + Array.Empty(), + new[] { provenance }); + + var flag = new PsirtFlagRecord( + advisoryKey, + "Oracle", + sourceName, + dto.AdvisoryId, + recordedAt.ToUniversalTime()); + + return (advisory, flag); + } + + private static IEnumerable BuildReferences(OracleDto dto, AdvisoryProvenance provenance) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var entries = new List<(AdvisoryReference Reference, int Priority)> + { + (new AdvisoryReference(dto.DetailUrl, "advisory", "oracle", null, provenance), 0), + }; + + foreach (var url in dto.References) + { + entries.Add((new AdvisoryReference(url, "reference", null, null, provenance), 1)); + } + + return entries + .GroupBy(tuple => tuple.Reference.Url, comparer) + .Select(group => group + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .First()) + .OrderBy(t => t.Priority) + .ThenBy(t => t.Reference.Kind ?? string.Empty, comparer) + .ThenBy(t => t.Reference.Url, comparer) + .Select(t => t.Reference); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs new file mode 100644 index 00000000..4f766562 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +internal static class OracleParser +{ + private static readonly Regex AnchorRegex = new("]+href=\"(?https?://[^\"]+)\"", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex TagRegex = new("<[^>]+>", RegexOptions.Compiled); + private static readonly Regex WhitespaceRegex = new("\\s+", RegexOptions.Compiled); + + public static OracleDto Parse(string html, OracleDocumentMetadata metadata) + { + ArgumentException.ThrowIfNullOrEmpty(html); + ArgumentNullException.ThrowIfNull(metadata); + + var content = Sanitize(html); + var references = ExtractReferences(html); + + return new OracleDto( + metadata.AdvisoryId, + metadata.Title, + metadata.DetailUri.ToString(), + metadata.Published, + content, + references); + } + + private static string Sanitize(string html) + { + var withoutTags = TagRegex.Replace(html, " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags) ?? string.Empty; + return WhitespaceRegex.Replace(decoded, " ").Trim(); + } + + private static IReadOnlyList ExtractReferences(string html) + { + var references = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in AnchorRegex.Matches(html)) + { + if (match.Success) + { + references.Add(match.Groups["url"].Value.Trim()); + } + } + + return references.Count == 0 + ? Array.Empty() + : references.OrderBy(url => url, StringComparer.OrdinalIgnoreCase).ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs new file mode 100644 index 00000000..2edd7b0c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs @@ -0,0 +1,295 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Feedser.Source.Vndr.Oracle.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public sealed class OracleConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly OracleOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public OracleConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IPsirtFlagStore psirtFlagStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VndrOracleConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + var now = _timeProvider.GetUtcNow(); + + foreach (var uri in _options.AdvisoryUris) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var advisoryId = DeriveAdvisoryId(uri); + var title = advisoryId.Replace('-', ' '); + var published = now; + + var metadata = OracleDocumentMetadata.CreateMetadata(advisoryId, title, published); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest(OracleOptions.HttpClientName, SourceName, uri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "text/html", "application/xhtml+xml", "text/plain;q=0.5" }, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + if (!pendingDocuments.Contains(result.Document.Id)) + { + pendingDocuments.Add(result.Document.Id); + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle fetch failed for {Uri}", uri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastProcessed(now); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("Oracle document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + OracleDto dto; + try + { + var metadata = OracleDocumentMetadata.FromDocument(document); + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = System.Text.Encoding.UTF8.GetString(content); + dto = OracleParser.Parse(html, metadata); + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = BsonDocument.Parse(json); + var validatedAt = _timeProvider.GetUtcNow(); + + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "oracle.advisory.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "oracle.advisory.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + OracleDto? dto; + try + { + var json = dtoRecord.Payload.ToJson(); + dto = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Oracle DTO deserialization failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("Oracle DTO payload deserialized as null for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var mappedAt = _timeProvider.GetUtcNow(); + var (advisory, flag) = OracleMapper.Map(dto, SourceName, mappedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return OracleCursor.FromBson(record?.Cursor); + } + + private async Task UpdateCursorAsync(OracleCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private static string DeriveAdvisoryId(Uri uri) + { + var segments = uri.Segments; + if (segments.Length == 0) + { + return uri.AbsoluteUri; + } + + var slug = segments[^1].Trim('/'); + if (string.IsNullOrWhiteSpace(slug)) + { + return uri.AbsoluteUri; + } + + return slug.Replace('.', '-'); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs new file mode 100644 index 00000000..1c7ce3fd --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Feedser.Source.Vndr.Oracle.Internal; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public static class OracleServiceCollectionExtensions +{ + public static IServiceCollection AddOracleConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(OracleOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Feedser.Oracle/1.0"; + clientOptions.AllowedHosts.Clear(); + foreach (var uri in options.AdvisoryUris) + { + clientOptions.AllowedHosts.Add(uri.Host); + } + }); + + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj b/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj new file mode 100644 index 00000000..8b8fb3b3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md new file mode 100644 index 00000000..2fb4ddb4 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md @@ -0,0 +1,12 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Oracle options & HttpClient configuration|BE-Conn-Oracle|Source.Common|TODO – add options with base URLs and allowlisted HttpClient configuration.| +|CPU calendar plus advisory fetchers|BE-Conn-Oracle|Source.Common|TODO – implement CPU calendar scraper and detail fetch pipeline.| +|Extractor for products/components/fix levels|BE-Conn-Oracle|Source.Common|TODO – normalize product/component names with fixed patch info.| +|DTO schema and validation|BE-Conn-Oracle, QA|Source.Common|TODO – define DTO + validation to quarantine malformed rows.| +|Canonical mapping with psirt_flags|BE-Conn-Oracle|Models|TODO – map advisories with psirt flags and vendor metadata.| +|SourceState and dedupe|BE-Conn-Oracle|Storage.Mongo|TODO – persist cursor/backoff and hash-based dedupe.| +|Golden fixtures and precedence tests (later with merge)|QA|Source.Vndr.Oracle|TODO – add fixtures verifying PSIRT overrides NVD ranges.| +|Dependency injection routine & job registration|BE-Conn-Oracle|Core|TODO – add DI routine plus scheduler registrations similar to other connectors.| +|Implement Oracle connector skeleton|BE-Conn-Oracle|Source.Common|TODO – replace stub plugin with full pipeline for fetching, DTO storage, and mapping.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs new file mode 100644 index 00000000..d22c9c27 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Oracle; + +public sealed class VndrOracleConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-oracle"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj new file mode 100644 index 00000000..3f55e9bc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs new file mode 100644 index 00000000..b88d72fc --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using MongoDB.Bson; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Vndr.Vmware; +using StellaOps.Feedser.Source.Vndr.Vmware.Internal; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Tests; + +public sealed class VmwareMapperTests +{ + [Fact] + public void Map_CreatesCanonicalAdvisory() + { + var modified = DateTimeOffset.UtcNow; + var dto = new VmwareDetailDto + { + AdvisoryId = "VMSA-2025-0001", + Title = "Sample VMware Advisory", + Summary = "Summary text", + Published = modified.AddDays(-1), + Modified = modified, + CveIds = new[] { "CVE-2025-0001", "CVE-2025-0002" }, + References = new[] + { + new VmwareReferenceDto { Url = "https://kb.vmware.com/some-kb", Type = "KB" }, + new VmwareReferenceDto { Url = "https://vmsa.vmware.com/vmsa/KB", Type = "Advisory" }, + }, + Affected = new[] + { + new VmwareAffectedProductDto + { + Product = "VMware vCenter", + Version = "7.0", + FixedVersion = "7.0u3" + } + } + }; + + var document = new DocumentRecord( + Guid.NewGuid(), + VmwareConnectorPlugin.SourceName, + "https://vmsa.vmware.com/vmsa/VMSA-2025-0001", + DateTimeOffset.UtcNow, + "sha256", + DocumentStatuses.PendingParse, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) + { + ["vmware.id"] = dto.AdvisoryId, + }, + null, + modified, + null, + null); + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + })); + + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, VmwareConnectorPlugin.SourceName, "vmware.v1", payload, DateTimeOffset.UtcNow); + + var advisory = VmwareMapper.Map(dto, document, dtoRecord); + + Assert.Equal(dto.AdvisoryId, advisory.AdvisoryKey); + Assert.Contains("CVE-2025-0001", advisory.Aliases); + Assert.Contains("CVE-2025-0002", advisory.Aliases); + Assert.Single(advisory.AffectedPackages); + Assert.Equal("VMware vCenter", advisory.AffectedPackages[0].Identifier); + Assert.Single(advisory.AffectedPackages[0].VersionRanges); + Assert.Equal("7.0", advisory.AffectedPackages[0].VersionRanges[0].IntroducedVersion); + Assert.Equal("7.0u3", advisory.AffectedPackages[0].VersionRanges[0].FixedVersion); + Assert.Equal(2, advisory.References.Length); + Assert.Equal("https://kb.vmware.com/some-kb", advisory.References[0].Url); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md b/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md new file mode 100644 index 00000000..5f1b714e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +VMware/Broadcom PSIRT connector ingesting VMSA advisories; authoritative for VMware products; maps affected versions/builds and emits psirt_flags. +## Scope +- Discover/fetch VMSA index and detail pages via Broadcom portal; window by advisory ID/date; follow updates/revisions. +- Validate HTML or JSON; extract CVEs, affected product versions/builds, workarounds, fixed versions; normalize product naming. +- Persist raw docs with sha256; manage source_state; idempotent mapping. +## Participants +- Source.Common (HTTP, cookies/session handling if needed, validators). +- Storage.Mongo (document, dto, advisory, alias, affected, reference, psirt_flags, source_state). +- Models (canonical). +- Core/WebService (jobs: source:vmware:fetch|parse|map). +- Merge engine (later) to prefer PSIRT ranges for VMware products. +## Interfaces & contracts +- Aliases: VMSA-YYYY-NNNN plus CVEs. +- Affected entries include Vendor=VMware, Product plus component; Versions carry fixed/fixedBy; tags may include build numbers or ESXi/VC levels. +- References: advisory URL, KBs, workaround pages; typed; deduped. +- Provenance: method=parser; value=VMSA id. +## In/Out of scope +In: PSIRT precedence mapping, affected/fixedBy extraction, advisory references. +Out: customer portal authentication flows beyond public advisories; downloading patches. +## Observability & security expectations +- Metrics: vmware.fetch.items, vmware.parse.fail, vmware.map.affected_count. +- Logs: vmsa ids, product counts, extraction timings; handle portal rate limits politely. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Vmware.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs new file mode 100644 index 00000000..593f37be --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs @@ -0,0 +1,54 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Configuration; + +public sealed class VmwareOptions +{ + public const string HttpClientName = "source.vmware"; + + public Uri IndexUri { get; set; } = new("https://example.invalid/vmsa/index.json", UriKind.Absolute); + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(2); + + public int MaxAdvisoriesPerFetch { get; set; } = 50; + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(2); + + [MemberNotNull(nameof(IndexUri))] + public void Validate() + { + if (IndexUri is null || !IndexUri.IsAbsoluteUri) + { + throw new InvalidOperationException("VMware index URI must be absolute."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("Initial backfill must be positive."); + } + + if (ModifiedTolerance < TimeSpan.Zero) + { + throw new InvalidOperationException("Modified tolerance cannot be negative."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException("Max advisories per fetch must be greater than zero."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("Request delay cannot be negative."); + } + + if (HttpTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("HTTP timeout must be positive."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs new file mode 100644 index 00000000..08791f8c --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal sealed record VmwareCursor( + DateTimeOffset? LastModified, + IReadOnlyCollection ProcessedIds, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyCollection EmptyStringList = Array.Empty(); + + public static VmwareCursor Empty { get; } = new(null, EmptyStringList, EmptyGuidList, EmptyGuidList); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastModified.HasValue) + { + document["lastModified"] = LastModified.Value.UtcDateTime; + } + + if (ProcessedIds.Count > 0) + { + document["processedIds"] = new BsonArray(ProcessedIds); + } + + return document; + } + + public static VmwareCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastModified = document.TryGetValue("lastModified", out var value) + ? ParseDate(value) + : null; + + var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray idsArray + ? idsArray.OfType().Where(static x => x.BsonType == BsonType.String).Select(static x => x.AsString).ToArray() + : EmptyStringList; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new VmwareCursor(lastModified, processedIds, pendingDocuments, pendingMappings); + } + + public VmwareCursor WithLastModified(DateTimeOffset timestamp, IEnumerable processedIds) + => this with + { + LastModified = timestamp.ToUniversalTime(), + ProcessedIds = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() ?? EmptyStringList, + }; + + public VmwareCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public VmwareCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public VmwareCursor AddProcessedId(string id) + { + if (string.IsNullOrWhiteSpace(id)) + { + return this; + } + + var set = new HashSet(ProcessedIds, StringComparer.OrdinalIgnoreCase) { id.Trim() }; + return this with { ProcessedIds = set.ToArray() }; + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs new file mode 100644 index 00000000..5a0b1063 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal sealed record VmwareDetailDto +{ + [JsonPropertyName("id")] + public string AdvisoryId { get; init; } = string.Empty; + + [JsonPropertyName("title")] + public string Title { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("cves")] + public IReadOnlyList? CveIds { get; init; } + + [JsonPropertyName("affected")] + public IReadOnlyList? Affected { get; init; } + + [JsonPropertyName("references")] + public IReadOnlyList? References { get; init; } +} + +internal sealed record VmwareAffectedProductDto +{ + [JsonPropertyName("product")] + public string Product { get; init; } = string.Empty; + + [JsonPropertyName("version")] + public string? Version { get; init; } + + [JsonPropertyName("fixedVersion")] + public string? FixedVersion { get; init; } +} + +internal sealed record VmwareReferenceDto +{ + [JsonPropertyName("type")] + public string? Type { get; init; } + + [JsonPropertyName("url")] + public string Url { get; init; } = string.Empty; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs new file mode 100644 index 00000000..de92c96e --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs @@ -0,0 +1,16 @@ +using System; +using System.Text.Json.Serialization; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal sealed record VmwareIndexItem +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("url")] + public string DetailUrl { get; init; } = string.Empty; + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs new file mode 100644 index 00000000..4084c6fb --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; + +internal static class VmwareMapper +{ + public static Advisory Map(VmwareDetailDto dto, DocumentRecord document, DtoRecord dtoRecord) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + + var fetchProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt); + var mappingProvenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "mapping", dto.AdvisoryId, dtoRecord.ValidatedAt); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, dtoRecord.ValidatedAt); + var affectedPackages = BuildAffectedPackages(dto, dtoRecord.ValidatedAt); + + return new Advisory( + dto.AdvisoryId, + dto.Title, + dto.Summary, + language: "en", + dto.Published?.ToUniversalTime(), + dto.Modified?.ToUniversalTime(), + severity: null, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mappingProvenance }); + } + + private static IEnumerable BuildAliases(VmwareDetailDto dto) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase) { dto.AdvisoryId }; + if (dto.CveIds is not null) + { + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + set.Add(cve.Trim()); + } + } + } + + return set; + } + + private static IReadOnlyList BuildReferences(VmwareDetailDto dto, DateTimeOffset recordedAt) + { + if (dto.References is null || dto.References.Count == 0) + { + return Array.Empty(); + } + + var references = new List(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + var kind = NormalizeReferenceKind(reference.Type); + var provenance = new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "reference", reference.Url, recordedAt); + try + { + references.Add(new AdvisoryReference(reference.Url, kind, reference.Type, null, provenance)); + } + catch (ArgumentException) + { + // ignore invalid urls + } + } + + references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); + return references.Count == 0 ? Array.Empty() : references; + } + + private static string? NormalizeReferenceKind(string? type) + { + if (string.IsNullOrWhiteSpace(type)) + { + return null; + } + + return type.Trim().ToLowerInvariant() switch + { + "advisory" => "advisory", + "kb" or "kb_article" => "kb", + "patch" => "patch", + "workaround" => "workaround", + _ => null, + }; + } + + private static IReadOnlyList BuildAffectedPackages(VmwareDetailDto dto, DateTimeOffset recordedAt) + { + if (dto.Affected is null || dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Affected.Count); + foreach (var product in dto.Affected) + { + if (string.IsNullOrWhiteSpace(product.Product)) + { + continue; + } + + var provenance = new[] + { + new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "affected", product.Product, recordedAt), + }; + + var ranges = new List(); + if (!string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.FixedVersion)) + { + ranges.Add(new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: product.Version, + fixedVersion: product.FixedVersion, + lastAffectedVersion: null, + rangeExpression: product.Version, + provenance: new AdvisoryProvenance(VmwareConnectorPlugin.SourceName, "range", product.Product, recordedAt))); + } + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Vendor, + product.Product, + platform: null, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: provenance)); + } + + return packages; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs new file mode 100644 index 00000000..e91474b8 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs @@ -0,0 +1,46 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +internal static class VmwareJobKinds +{ + public const string Fetch = "source:vmware:fetch"; + public const string Parse = "source:vmware:parse"; + public const string Map = "source:vmware:map"; +} + +internal sealed class VmwareFetchJob : IJob +{ + private readonly VmwareConnector _connector; + + public VmwareFetchJob(VmwareConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class VmwareParseJob : IJob +{ + private readonly VmwareConnector _connector; + + public VmwareParseJob(VmwareConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class VmwareMapJob : IJob +{ + private readonly VmwareConnector _connector; + + public VmwareMapJob(VmwareConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj b/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj new file mode 100644 index 00000000..08a26244 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + <_Parameter1>StellaOps.Feedser.Tests + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md b/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md new file mode 100644 index 00000000..e35ff49f --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md @@ -0,0 +1,16 @@ +# Source.Vndr.Vmware — Task Board + +| ID | Task | Owner | Status | Depends On | Notes | +|------|-----------------------------------------------|-------|--------|------------|-------| +| VM1 | Advisory listing discovery + cursor | Conn | TODO | Common | Track revisions; respect VMware PSIRT cadence. | +| VM2 | VMSA parser → DTO | QA | TODO | | Extract product/version/CVE/severity; capture fixed build numbers. | +| VM3 | Canonical mapping (aliases/affected/refs) | Conn | TODO | Models | Deterministic ordering; set vendor="VMware" and advisory_id_text=VMSA. | +| VM4 | Snapshot tests + resume | QA | TODO | Storage | | +| VM5 | Observability | QA | TODO | | Metrics counters. | +| VM6 | SourceState + hash dedupe | Conn | TODO | Storage | Skip unchanged advisories; ensure idempotent reruns. | +| VM6a | Options & HttpClient configuration | Conn | TODO | Source.Common | Introduce `VmwareOptions` with base portal URLs and allowlisted HttpClient setup. | +| VM7 | Dependency injection routine & scheduler registration | Conn | TODO | Core | Wire HttpClient/options and register fetch/parse/map jobs consistent with other connectors. | +| VM8 | Replace stub plugin with connector pipeline skeleton | Conn | TODO | Source.Common | Implement fetch/parse/map scaffolding persisting source_state, documents, and canonical advisories. | + +## Changelog +- YYYY-MM-DD: Created. diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs new file mode 100644 index 00000000..3ba2db7b --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs @@ -0,0 +1,374 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; +using StellaOps.Feedser.Source.Vndr.Vmware.Internal; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +public sealed class VmwareConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly VmwareOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public VmwareConnector( + IHttpClientFactory httpClientFactory, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VmwareConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var remainingCapacity = _options.MaxAdvisoriesPerFetch; + + IReadOnlyList indexItems; + try + { + indexItems = await FetchIndexAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to retrieve VMware advisory index"); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (indexItems.Count == 0) + { + return; + } + + var orderedItems = indexItems + .Where(static item => !string.IsNullOrWhiteSpace(item.Id) && !string.IsNullOrWhiteSpace(item.DetailUrl)) + .OrderBy(static item => item.Modified ?? DateTimeOffset.MinValue) + .ThenBy(static item => item.Id, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var baseline = cursor.LastModified ?? now - _options.InitialBackfill; + var processedIds = new HashSet(cursor.ProcessedIds, StringComparer.OrdinalIgnoreCase); + var maxModified = cursor.LastModified ?? DateTimeOffset.MinValue; + var processedUpdated = false; + + foreach (var item in orderedItems) + { + if (remainingCapacity <= 0) + { + break; + } + + cancellationToken.ThrowIfCancellationRequested(); + + var modified = (item.Modified ?? DateTimeOffset.MinValue).ToUniversalTime(); + if (modified < baseline - _options.ModifiedTolerance) + { + continue; + } + + if (cursor.LastModified.HasValue && modified < cursor.LastModified.Value - _options.ModifiedTolerance) + { + continue; + } + + if (modified == cursor.LastModified && cursor.ProcessedIds.Contains(item.Id, StringComparer.OrdinalIgnoreCase)) + { + continue; + } + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["vmware.id"] = item.Id, + ["vmware.modified"] = modified.ToString("O"), + }; + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync( + new SourceFetchRequest(VmwareOptions.HttpClientName, SourceName, new Uri(item.DetailUrl)) + { + Metadata = metadata, + }, + cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to fetch VMware advisory {AdvisoryId}", item.Id); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.IsNotModified) + { + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + pendingDocuments.Add(result.Document.Id); + remainingCapacity--; + + if (modified > maxModified) + { + maxModified = modified; + processedIds.Clear(); + processedUpdated = true; + } + + if (modified == maxModified) + { + processedIds.Add(item.Id); + processedUpdated = true; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(cursor.PendingMappings); + + if (processedUpdated) + { + updatedCursor = updatedCursor.WithLastModified(maxModified, processedIds); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remaining = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remaining.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("VMware document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed downloading VMware document {DocumentId}", document.Id); + throw; + } + + VmwareDetailDto? detail; + try + { + detail = JsonSerializer.Deserialize(bytes, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to deserialize VMware advisory {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId)) + { + _logger.LogWarning("VMware advisory document {DocumentId} contained empty payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remaining.Remove(documentId); + continue; + } + + var sanitized = JsonSerializer.Serialize(detail, SerializerOptions); + var payload = MongoDB.Bson.BsonDocument.Parse(sanitized); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "vmware.v1", payload, _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remaining.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remaining) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dto = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dto is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var json = dto.Payload.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + }); + + VmwareDetailDto? detail; + try + { + detail = JsonSerializer.Deserialize(json, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize VMware DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (detail is null || string.IsNullOrWhiteSpace(detail.AdvisoryId)) + { + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = VmwareMapper.Map(detail, document, dto); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task> FetchIndexAsync(CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(VmwareOptions.HttpClientName); + using var response = await client.GetAsync(_options.IndexUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var items = await JsonSerializer.DeserializeAsync>(stream, SerializerOptions, cancellationToken).ConfigureAwait(false); + return items ?? Array.Empty(); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? VmwareCursor.Empty : VmwareCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(VmwareCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs new file mode 100644 index 00000000..58d782a3 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs @@ -0,0 +1,20 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +public sealed class VmwareConnectorPlugin : IConnectorPlugin +{ + public string Name => SourceName; + + public static string SourceName => "vmware"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs new file mode 100644 index 00000000..6ecb1943 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs @@ -0,0 +1,53 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +public sealed class VmwareDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "feedser:sources:vmware"; + private const string FetchCron = "10,40 * * * *"; + private const string ParseCron = "15,45 * * * *"; + private const string MapCron = "20,50 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(10); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(15); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(5); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddVmwareConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob( + VmwareJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob( + VmwareJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob( + VmwareJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs new file mode 100644 index 00000000..32018526 --- /dev/null +++ b/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; + +namespace StellaOps.Feedser.Source.Vndr.Vmware; + +public static class VmwareServiceCollectionExtensions +{ + public static IServiceCollection AddVmwareConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(VmwareOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = new Uri(options.IndexUri.GetLeftPart(UriPartial.Authority)); + clientOptions.Timeout = options.HttpTimeout; + clientOptions.UserAgent = "StellaOps.Feedser.VMware/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.IndexUri.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs new file mode 100644 index 00000000..0639e096 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs @@ -0,0 +1,187 @@ +using System.Diagnostics; +using System.Linq; +using System.Threading; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.Migrations; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class AdvisoryStorePerformanceTests : IClassFixture +{ + private const int LargeAdvisoryCount = 30; + private const int AliasesPerAdvisory = 24; + private const int ReferencesPerAdvisory = 180; + private const int AffectedPackagesPerAdvisory = 140; + private const int VersionRangesPerPackage = 4; + private const int CvssMetricsPerAdvisory = 24; + private const int ProvenanceEntriesPerAdvisory = 16; + private static readonly string LargeSummary = new('A', 128 * 1024); + private static readonly DateTimeOffset BasePublished = new(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + private static readonly DateTimeOffset BaseRecorded = new(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + private static readonly TimeSpan TotalBudget = TimeSpan.FromSeconds(28); + private const double UpsertBudgetPerAdvisoryMs = 500; + private const double FetchBudgetPerAdvisoryMs = 200; + private const double FindBudgetPerAdvisoryMs = 200; + + private readonly MongoIntegrationFixture _fixture; + private readonly ITestOutputHelper _output; + + public AdvisoryStorePerformanceTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + [Fact] + public async Task UpsertAndQueryLargeAdvisories_CompletesWithinBudget() + { + var databaseName = $"feedser-performance-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + + try + { + var migrationRunner = new MongoMigrationRunner( + database, + Array.Empty(), + NullLogger.Instance, + TimeProvider.System); + + var bootstrapper = new MongoBootstrapper( + database, + Options.Create(new MongoStorageOptions()), + NullLogger.Instance, + migrationRunner); + await bootstrapper.InitializeAsync(CancellationToken.None); + + var store = new AdvisoryStore(database, NullLogger.Instance); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(45)); + + var advisories = Enumerable.Range(0, LargeAdvisoryCount) + .Select(CreateLargeAdvisory) + .ToArray(); + + var upsertWatch = Stopwatch.StartNew(); + foreach (var advisory in advisories) + { + await store.UpsertAsync(advisory, cts.Token); + } + + upsertWatch.Stop(); + var upsertPerAdvisory = upsertWatch.Elapsed.TotalMilliseconds / LargeAdvisoryCount; + + var fetchWatch = Stopwatch.StartNew(); + var recent = await store.GetRecentAsync(LargeAdvisoryCount, cts.Token); + fetchWatch.Stop(); + var fetchPerAdvisory = fetchWatch.Elapsed.TotalMilliseconds / LargeAdvisoryCount; + + Assert.Equal(LargeAdvisoryCount, recent.Count); + + var findWatch = Stopwatch.StartNew(); + foreach (var advisory in advisories) + { + var fetched = await store.FindAsync(advisory.AdvisoryKey, cts.Token); + Assert.NotNull(fetched); + } + + findWatch.Stop(); + var findPerAdvisory = findWatch.Elapsed.TotalMilliseconds / LargeAdvisoryCount; + + var totalElapsed = upsertWatch.Elapsed + fetchWatch.Elapsed + findWatch.Elapsed; + + _output.WriteLine($"Upserted {LargeAdvisoryCount} large advisories in {upsertWatch.Elapsed} ({upsertPerAdvisory:F2} ms/doc)."); + _output.WriteLine($"Fetched recent advisories in {fetchWatch.Elapsed} ({fetchPerAdvisory:F2} ms/doc)."); + _output.WriteLine($"Looked up advisories individually in {findWatch.Elapsed} ({findPerAdvisory:F2} ms/doc)."); + _output.WriteLine($"Total elapsed {totalElapsed}."); + + Assert.True(upsertPerAdvisory <= UpsertBudgetPerAdvisoryMs, $"Upsert exceeded {UpsertBudgetPerAdvisoryMs} ms per advisory: {upsertPerAdvisory:F2} ms."); + Assert.True(fetchPerAdvisory <= FetchBudgetPerAdvisoryMs, $"GetRecent exceeded {FetchBudgetPerAdvisoryMs} ms per advisory: {fetchPerAdvisory:F2} ms."); + Assert.True(findPerAdvisory <= FindBudgetPerAdvisoryMs, $"Find exceeded {FindBudgetPerAdvisoryMs} ms per advisory: {findPerAdvisory:F2} ms."); + Assert.True(totalElapsed <= TotalBudget, $"Mongo advisory operations exceeded total budget {TotalBudget}: {totalElapsed}."); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + private static Advisory CreateLargeAdvisory(int index) + { + var baseKey = $"ADV-LARGE-{index:D4}"; + var published = BasePublished.AddDays(index); + var modified = published.AddHours(6); + + var aliases = Enumerable.Range(0, AliasesPerAdvisory) + .Select(i => $"ALIAS-{baseKey}-{i:D4}") + .ToArray(); + + var provenance = Enumerable.Range(0, ProvenanceEntriesPerAdvisory) + .Select(i => new AdvisoryProvenance( + source: i % 2 == 0 ? "nvd" : "vendor", + kind: i % 3 == 0 ? "normalized" : "enriched", + value: $"prov-{baseKey}-{i:D3}", + recordedAt: BaseRecorded.AddDays(i))) + .ToArray(); + + var references = Enumerable.Range(0, ReferencesPerAdvisory) + .Select(i => new AdvisoryReference( + url: $"https://vuln.example.com/{baseKey}/ref/{i:D4}", + kind: i % 2 == 0 ? "advisory" : "article", + sourceTag: $"tag-{i % 7}", + summary: $"Reference {baseKey} #{i}", + provenance: provenance[i % provenance.Length])) + .ToArray(); + + var affectedPackages = Enumerable.Range(0, AffectedPackagesPerAdvisory) + .Select(i => new AffectedPackage( + type: i % 3 == 0 ? AffectedPackageTypes.Rpm : AffectedPackageTypes.Deb, + identifier: $"pkg/{baseKey}/{i:D4}", + platform: i % 4 == 0 ? "linux/x86_64" : "linux/aarch64", + versionRanges: Enumerable.Range(0, VersionRangesPerPackage) + .Select(r => new AffectedVersionRange( + rangeKind: r % 2 == 0 ? "semver" : "evr", + introducedVersion: $"1.{index}.{i}.{r}", + fixedVersion: $"2.{index}.{i}.{r}", + lastAffectedVersion: $"1.{index}.{i}.{r}", + rangeExpression: $">=1.{index}.{i}.{r} <2.{index}.{i}.{r}", + provenance: provenance[(i + r) % provenance.Length])) + .ToArray(), + statuses: Array.Empty(), + provenance: new[] + { + provenance[i % provenance.Length], + provenance[(i + 3) % provenance.Length], + })) + .ToArray(); + + var cvssMetrics = Enumerable.Range(0, CvssMetricsPerAdvisory) + .Select(i => new CvssMetric( + version: i % 2 == 0 ? "3.1" : "2.0", + vector: $"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:{(i % 3 == 0 ? "H" : "L")}", + baseScore: Math.Max(0, 9.8 - i * 0.2), + baseSeverity: i % 3 == 0 ? "critical" : "high", + provenance: provenance[i % provenance.Length])) + .ToArray(); + + return new Advisory( + advisoryKey: baseKey, + title: $"Large advisory {baseKey}", + summary: LargeSummary, + language: "en", + published: published, + modified: modified, + severity: "critical", + exploitKnown: index % 2 == 0, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: cvssMetrics, + provenance: provenance); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs new file mode 100644 index 00000000..848f536f --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStoreTests.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Models; +using StellaOps.Feedser.Storage.Mongo.Advisories; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class AdvisoryStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public AdvisoryStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndFetchAdvisory() + { + var store = new AdvisoryStore(_fixture.Database, NullLogger.Instance); + var advisory = new Advisory( + advisoryKey: "ADV-1", + title: "Sample Advisory", + summary: "Demo", + language: "en", + published: DateTimeOffset.UtcNow, + modified: DateTimeOffset.UtcNow, + severity: "medium", + exploitKnown: false, + aliases: new[] { "ALIAS-1" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + await store.UpsertAsync(advisory, CancellationToken.None); + + var fetched = await store.FindAsync("ADV-1", CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal(advisory.AdvisoryKey, fetched!.AdvisoryKey); + + var recent = await store.GetRecentAsync(5, CancellationToken.None); + Assert.NotEmpty(recent); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs new file mode 100644 index 00000000..a8eef366 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs @@ -0,0 +1,51 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Storage.Mongo.Documents; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class DocumentStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public DocumentStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndLookupDocument() + { + var store = new DocumentStore(_fixture.Database, NullLogger.Instance); + var id = Guid.NewGuid(); + var record = new DocumentRecord( + id, + "source", + "https://example.com/advisory.json", + DateTimeOffset.UtcNow, + "sha123", + "pending", + "application/json", + new Dictionary { ["etag"] = "abc" }, + new Dictionary { ["note"] = "test" }, + "etag-value", + DateTimeOffset.UtcNow, + null, + DateTimeOffset.UtcNow.AddDays(30)); + + var upserted = await store.UpsertAsync(record, CancellationToken.None); + Assert.Equal(id, upserted.Id); + + var fetched = await store.FindBySourceAndUriAsync("source", "https://example.com/advisory.json", CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal("pending", fetched!.Status); + Assert.Equal("test", fetched.Metadata!["note"]); + + var statusUpdated = await store.UpdateStatusAsync(id, "processed", CancellationToken.None); + Assert.True(statusUpdated); + + var refreshed = await store.FindAsync(id, CancellationToken.None); + Assert.NotNull(refreshed); + Assert.Equal("processed", refreshed!.Status); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs new file mode 100644 index 00000000..7667dcb5 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Bson; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class DtoStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public DtoStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndLookupDto() + { + var store = new DtoStore(_fixture.Database, NullLogger.Instance); + var record = new DtoRecord( + Guid.NewGuid(), + Guid.NewGuid(), + "source", + "1.0", + new BsonDocument("value", 1), + DateTimeOffset.UtcNow); + + var upserted = await store.UpsertAsync(record, CancellationToken.None); + Assert.Equal(record.DocumentId, upserted.DocumentId); + + var fetched = await store.FindByDocumentIdAsync(record.DocumentId, CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal(1, fetched!.Payload["value"].AsInt32); + + var bySource = await store.GetBySourceAsync("source", 10, CancellationToken.None); + Assert.Single(bySource); + Assert.Equal(record.DocumentId, bySource[0].DocumentId); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs new file mode 100644 index 00000000..c7bd6ae4 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +public sealed class ExportStateManagerTests +{ + [Fact] + public async Task StoreFullExportInitializesBaseline() + { + var store = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-20T12:00:00Z")); + var manager = new ExportStateManager(store, timeProvider); + + var record = await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T120000Z", + exportDigest: "sha256:abcd", + cursor: "cursor-1", + targetRepository: "registry.local/json", + exporterVersion: "1.0.0", + cancellationToken: CancellationToken.None); + + Assert.Equal("export:json", record.Id); + Assert.Equal("20240720T120000Z", record.BaseExportId); + Assert.Equal("sha256:abcd", record.BaseDigest); + Assert.Equal("sha256:abcd", record.LastFullDigest); + Assert.Null(record.LastDeltaDigest); + Assert.Equal("cursor-1", record.ExportCursor); + Assert.Equal("registry.local/json", record.TargetRepository); + Assert.Equal("1.0.0", record.ExporterVersion); + Assert.Equal(timeProvider.Now, record.UpdatedAt); + } + + [Fact] + public async Task StoreDeltaExportRequiresBaseline() + { + var store = new InMemoryExportStateStore(); + var manager = new ExportStateManager(store); + + await Assert.ThrowsAsync(() => manager.StoreDeltaExportAsync( + exporterId: "export:json", + deltaDigest: "sha256:def", + cursor: null, + exporterVersion: "1.0.1", + cancellationToken: CancellationToken.None)); + } + + [Fact] + public async Task StoreDeltaExportUpdatesExistingState() + { + var store = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-20T12:00:00Z")); + var manager = new ExportStateManager(store, timeProvider); + + await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T120000Z", + exportDigest: "sha256:abcd", + cursor: "cursor-1", + targetRepository: null, + exporterVersion: "1.0.0", + cancellationToken: CancellationToken.None); + + timeProvider.Advance(TimeSpan.FromMinutes(10)); + var delta = await manager.StoreDeltaExportAsync( + exporterId: "export:json", + deltaDigest: "sha256:ef01", + cursor: "cursor-2", + exporterVersion: "1.0.1", + cancellationToken: CancellationToken.None); + + Assert.Equal("sha256:ef01", delta.LastDeltaDigest); + Assert.Equal("cursor-2", delta.ExportCursor); + Assert.Equal("1.0.1", delta.ExporterVersion); + Assert.Equal(timeProvider.Now, delta.UpdatedAt); + Assert.Equal("sha256:abcd", delta.LastFullDigest); + } + + private sealed class InMemoryExportStateStore : IExportStateStore + { + private readonly Dictionary _records = new(StringComparer.Ordinal); + + public Task FindAsync(string id, CancellationToken cancellationToken) + { + _records.TryGetValue(id, out var record); + return Task.FromResult(record); + } + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _records[record.Id] = record; + return Task.FromResult(record); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + public TestTimeProvider(DateTimeOffset start) => Now = start; + + public DateTimeOffset Now { get; private set; } + + public void Advance(TimeSpan delta) => Now = Now.Add(delta); + + public override DateTimeOffset GetUtcNow() => Now; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs new file mode 100644 index 00000000..4f88562d --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs @@ -0,0 +1,38 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Storage.Mongo.Exporting; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class ExportStateStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public ExportStateStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndFetchExportState() + { + var store = new ExportStateStore(_fixture.Database, NullLogger.Instance); + var record = new ExportStateRecord( + Id: "json", + BaseExportId: "base", + BaseDigest: "sha-base", + LastFullDigest: "sha-full", + LastDeltaDigest: null, + ExportCursor: "cursor", + TargetRepository: "repo", + ExporterVersion: "1.0", + UpdatedAt: DateTimeOffset.UtcNow); + + var saved = await store.UpsertAsync(record, CancellationToken.None); + Assert.Equal("json", saved.Id); + + var fetched = await store.FindAsync("json", CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal("sha-full", fetched!.LastFullDigest); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs new file mode 100644 index 00000000..bb75d08c --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs @@ -0,0 +1,34 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class MergeEventStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public MergeEventStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task AppendAndReadMergeEvents() + { + var store = new MergeEventStore(_fixture.Database, NullLogger.Instance); + var record = new MergeEventRecord( + Guid.NewGuid(), + "ADV-1", + new byte[] { 0x01 }, + new byte[] { 0x02 }, + DateTimeOffset.UtcNow, + new List { Guid.NewGuid() }); + + await store.AppendAsync(record, CancellationToken.None); + + var recent = await store.GetRecentAsync("ADV-1", 10, CancellationToken.None); + Assert.Single(recent); + Assert.Equal(record.AfterHash, recent[0].AfterHash); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs new file mode 100644 index 00000000..44eb0535 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs @@ -0,0 +1,238 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Migrations; +using Xunit; + +namespace StellaOps.Feedser.Storage.Mongo.Tests.Migrations; + +[Collection("mongo-fixture")] +public sealed class MongoMigrationRunnerTests +{ + private readonly MongoIntegrationFixture _fixture; + + public MongoMigrationRunnerTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task RunAsync_AppliesPendingMigrationsOnce() + { + var databaseName = $"feedser-migrations-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var migration = new TestMigration(); + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + await runner.RunAsync(CancellationToken.None); + + Assert.Equal(1, migration.ApplyCount); + + var count = await database + .GetCollection(MongoStorageDefaults.Collections.Migrations) + .CountDocumentsAsync(FilterDefinition.Empty); + Assert.Equal(1, count); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureDocumentExpiryIndexesMigration_CreatesTtlIndexWhenRetentionEnabled() + { + var databaseName = $"feedser-doc-ttl-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Document); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var options = Options.Create(new MongoStorageOptions + { + RawDocumentRetention = TimeSpan.FromDays(45), + RawDocumentRetentionTtlGrace = TimeSpan.FromHours(12), + }); + + var migration = new EnsureDocumentExpiryIndexesMigration(options); + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var indexes = await database + .GetCollection(MongoStorageDefaults.Collections.Document) + .Indexes.ListAsync(); + var indexList = await indexes.ToListAsync(); + + var ttlIndex = indexList.Single(x => x["name"].AsString == "document_expiresAt_ttl"); + Assert.Equal(0, ttlIndex["expireAfterSeconds"].ToDouble()); + Assert.True(ttlIndex["partialFilterExpression"].AsBsonDocument["expiresAt"].AsBsonDocument["$exists"].ToBoolean()); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureDocumentExpiryIndexesMigration_DropsTtlIndexWhenRetentionDisabled() + { + var databaseName = $"feedser-doc-notl-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Document); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var collection = database.GetCollection(MongoStorageDefaults.Collections.Document); + var keys = Builders.IndexKeys.Ascending("expiresAt"); + var options = new CreateIndexOptions + { + Name = "document_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("expiresAt", true), + }; + + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options)); + + var migration = new EnsureDocumentExpiryIndexesMigration(Options.Create(new MongoStorageOptions + { + RawDocumentRetention = TimeSpan.Zero, + })); + + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var indexes = await collection.Indexes.ListAsync(); + var indexList = await indexes.ToListAsync(); + + Assert.DoesNotContain(indexList, x => x["name"].AsString == "document_expiresAt_ttl"); + var nonTtl = indexList.Single(x => x["name"].AsString == "document_expiresAt"); + Assert.False(nonTtl.Contains("expireAfterSeconds")); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureGridFsExpiryIndexesMigration_CreatesTtlIndexWhenRetentionEnabled() + { + var databaseName = $"feedser-gridfs-ttl-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync("documents.files"); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var migration = new EnsureGridFsExpiryIndexesMigration(Options.Create(new MongoStorageOptions + { + RawDocumentRetention = TimeSpan.FromDays(30), + })); + + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var indexes = await database.GetCollection("documents.files").Indexes.ListAsync(); + var indexList = await indexes.ToListAsync(); + + var ttlIndex = indexList.Single(x => x["name"].AsString == "gridfs_files_expiresAt_ttl"); + Assert.Equal(0, ttlIndex["expireAfterSeconds"].ToDouble()); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureGridFsExpiryIndexesMigration_DropsTtlIndexWhenRetentionDisabled() + { + var databaseName = $"feedser-gridfs-notl-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync("documents.files"); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var collection = database.GetCollection("documents.files"); + var keys = Builders.IndexKeys.Ascending("metadata.expiresAt"); + var options = new CreateIndexOptions + { + Name = "gridfs_files_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("metadata.expiresAt", true), + }; + + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options)); + + var migration = new EnsureGridFsExpiryIndexesMigration(Options.Create(new MongoStorageOptions + { + RawDocumentRetention = TimeSpan.Zero, + })); + + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var indexes = await collection.Indexes.ListAsync(); + var indexList = await indexes.ToListAsync(); + + Assert.DoesNotContain(indexList, x => x["name"].AsString == "gridfs_files_expiresAt_ttl"); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + private sealed class TestMigration : IMongoMigration + { + public int ApplyCount { get; private set; } + + public string Id => "999_test"; + + public string Description => "test migration"; + + public Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ApplyCount++; + return Task.CompletedTask; + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs new file mode 100644 index 00000000..271951ed --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs @@ -0,0 +1,100 @@ +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Storage.Mongo; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class MongoJobStoreTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public MongoJobStoreTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task CreateStartCompleteLifecycle() + { + var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); + var store = new MongoJobStore(collection, NullLogger.Instance); + + var request = new JobRunCreateRequest( + Kind: "mongo:test", + Trigger: "unit", + Parameters: new Dictionary { ["scope"] = "lifecycle" }, + ParametersHash: "abc", + Timeout: TimeSpan.FromSeconds(5), + LeaseDuration: TimeSpan.FromSeconds(2), + CreatedAt: DateTimeOffset.UtcNow); + + var created = await store.CreateAsync(request, CancellationToken.None); + Assert.Equal(JobRunStatus.Pending, created.Status); + + var started = await store.TryStartAsync(created.RunId, DateTimeOffset.UtcNow, CancellationToken.None); + Assert.NotNull(started); + Assert.Equal(JobRunStatus.Running, started!.Status); + + var completed = await store.TryCompleteAsync(created.RunId, new JobRunCompletion(JobRunStatus.Succeeded, DateTimeOffset.UtcNow, null), CancellationToken.None); + Assert.NotNull(completed); + Assert.Equal(JobRunStatus.Succeeded, completed!.Status); + + var recent = await store.GetRecentRunsAsync(null, 10, CancellationToken.None); + Assert.Single(recent); + Assert.Equal(JobRunStatus.Succeeded, recent[0].Status); + + var active = await store.GetActiveRunsAsync(CancellationToken.None); + Assert.Empty(active); + + var last = await store.GetLastRunAsync("mongo:test", CancellationToken.None); + Assert.NotNull(last); + Assert.Equal(completed.RunId, last!.RunId); + } + + [Fact] + public async Task StartAndFailRunHonorsStateTransitions() + { + var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); + var store = new MongoJobStore(collection, NullLogger.Instance); + + var request = new JobRunCreateRequest( + Kind: "mongo:failure", + Trigger: "unit", + Parameters: new Dictionary(), + ParametersHash: null, + Timeout: null, + LeaseDuration: null, + CreatedAt: DateTimeOffset.UtcNow); + + var created = await store.CreateAsync(request, CancellationToken.None); + var firstStart = await store.TryStartAsync(created.RunId, DateTimeOffset.UtcNow, CancellationToken.None); + Assert.NotNull(firstStart); + + // Second start attempt should be rejected once running. + var secondStart = await store.TryStartAsync(created.RunId, DateTimeOffset.UtcNow.AddSeconds(1), CancellationToken.None); + Assert.Null(secondStart); + + var failure = await store.TryCompleteAsync( + created.RunId, + new JobRunCompletion(JobRunStatus.Failed, DateTimeOffset.UtcNow.AddSeconds(2), "boom"), + CancellationToken.None); + + Assert.NotNull(failure); + Assert.Equal("boom", failure!.Error); + Assert.Equal(JobRunStatus.Failed, failure.Status); + } + + [Fact] + public async Task CompletingUnknownRunReturnsNull() + { + var collection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Jobs); + var store = new MongoJobStore(collection, NullLogger.Instance); + + var result = await store.TryCompleteAsync(Guid.NewGuid(), new JobRunCompletion(JobRunStatus.Succeeded, DateTimeOffset.UtcNow, null), CancellationToken.None); + + Assert.Null(result); + } +} + diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs new file mode 100644 index 00000000..40cbedb9 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs @@ -0,0 +1,55 @@ +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Bson; +using StellaOps.Feedser.Storage.Mongo; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class MongoSourceStateRepositoryTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public MongoSourceStateRepositoryTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task UpsertAndUpdateCursorFlow() + { + var repository = new MongoSourceStateRepository(_fixture.Database, NullLogger.Instance); + var sourceName = "nvd"; + + var record = new SourceStateRecord( + SourceName: sourceName, + Enabled: true, + Paused: false, + Cursor: new BsonDocument("page", 1), + LastSuccess: null, + LastFailure: null, + FailCount: 0, + BackoffUntil: null, + UpdatedAt: DateTimeOffset.UtcNow, + LastFailureReason: null); + + var upserted = await repository.UpsertAsync(record, CancellationToken.None); + Assert.True(upserted.Enabled); + + var cursor = new BsonDocument("page", 2); + var updated = await repository.UpdateCursorAsync(sourceName, cursor, DateTimeOffset.UtcNow, CancellationToken.None); + Assert.NotNull(updated); + Assert.Equal(0, updated!.FailCount); + Assert.Equal(2, updated.Cursor["page"].AsInt32); + + var failure = await repository.MarkFailureAsync(sourceName, DateTimeOffset.UtcNow, TimeSpan.FromMinutes(5), "network timeout", CancellationToken.None); + Assert.NotNull(failure); + Assert.Equal(1, failure!.FailCount); + Assert.NotNull(failure.BackoffUntil); + Assert.Equal("network timeout", failure.LastFailureReason); + + var fetched = await repository.TryGetAsync(sourceName, CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal(failure.BackoffUntil, fetched!.BackoffUntil); + Assert.Equal("network timeout", fetched.LastFailureReason); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs new file mode 100644 index 00000000..1c78adc5 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs @@ -0,0 +1,92 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class RawDocumentRetentionServiceTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public RawDocumentRetentionServiceTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task SweepExpiredDocumentsAsync_RemovesExpiredRawDocuments() + { + var database = _fixture.Database; + var documents = database.GetCollection(MongoStorageDefaults.Collections.Document); + var dtos = database.GetCollection(MongoStorageDefaults.Collections.Dto); + var bucket = new GridFSBucket(database, new GridFSBucketOptions { BucketName = "documents" }); + + var now = new DateTimeOffset(2024, 10, 1, 12, 0, 0, TimeSpan.Zero); + var fakeTime = new FakeTimeProvider(now); + + var options = Options.Create(new MongoStorageOptions + { + ConnectionString = _fixture.Runner.ConnectionString, + DatabaseName = database.DatabaseNamespace.DatabaseName, + RawDocumentRetention = TimeSpan.FromDays(1), + RawDocumentRetentionTtlGrace = TimeSpan.Zero, + RawDocumentRetentionSweepInterval = TimeSpan.FromMinutes(5), + }); + + var expiredId = Guid.NewGuid(); + var gridFsId = await bucket.UploadFromBytesAsync("expired", new byte[] { 1, 2, 3 }); + await documents.InsertOneAsync(new DocumentDocument + { + Id = expiredId, + SourceName = "nvd", + Uri = "https://example.test/cve", + FetchedAt = now.AddDays(-2).UtcDateTime, + Sha256 = "abc", + Status = "pending", + ExpiresAt = now.AddMinutes(-5).UtcDateTime, + GridFsId = gridFsId, + }); + + await dtos.InsertOneAsync(new DtoDocument + { + Id = Guid.NewGuid(), + DocumentId = expiredId, + SourceName = "nvd", + SchemaVersion = "schema", + Payload = new BsonDocument("value", 1), + ValidatedAt = now.UtcDateTime, + }); + + var freshId = Guid.NewGuid(); + await documents.InsertOneAsync(new DocumentDocument + { + Id = freshId, + SourceName = "nvd", + Uri = "https://example.test/future", + FetchedAt = now.UtcDateTime, + Sha256 = "def", + Status = "pending", + ExpiresAt = now.AddHours(1).UtcDateTime, + GridFsId = null, + }); + + var service = new RawDocumentRetentionService(database, options, NullLogger.Instance, fakeTime); + + var removed = await service.SweepExpiredDocumentsAsync(CancellationToken.None); + + Assert.Equal(1, removed); + Assert.Equal(0, await documents.CountDocumentsAsync(d => d.Id == expiredId)); + Assert.Equal(0, await dtos.CountDocumentsAsync(d => d.DocumentId == expiredId)); + Assert.Equal(1, await documents.CountDocumentsAsync(d => d.Id == freshId)); + + using var cursor = await bucket.FindAsync(Builders.Filter.Eq(info => info.Id, gridFsId)); + Assert.Empty(await cursor.ToListAsync()); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj b/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj new file mode 100644 index 00000000..cd30dc7f --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + + + + + + + diff --git a/src/StellaOps.Feedser.Storage.Mongo/AGENTS.md b/src/StellaOps.Feedser.Storage.Mongo/AGENTS.md new file mode 100644 index 00000000..bfcfeb64 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Canonical persistence for raw documents, DTOs, canonical advisories, jobs, and state. Provides repositories and bootstrapper for collections/indexes. +## Scope +- Collections (MongoStorageDefaults): source, source_state, document, dto, advisory, alias, affected, reference, kev_flag, ru_flags, jp_flags, psirt_flags, merge_event, export_state, locks, jobs; GridFS bucket fs.documents; field names include ttlAt (locks), sourceName, uri, advisoryKey. +- Records: SourceState (cursor, lastSuccess/error, failCount, backoffUntil), JobRun, MergeEvent, ExportState, Advisory documents mirroring Models with embedded arrays when practical. +- Bootstrapper: create collections, indexes (unique advisoryKey, scheme/value, platform/name, published, modified), TTL on locks, and validate connectivity for /ready health probes. +- Job store: create, read, mark completed/failed; compute durations; recent/last queries; active by status. +- Advisory store: CRUD for canonical advisories; query by key/alias and list for exporters with deterministic paging. +## Participants +- Core jobs read/write runs and leases; WebService /ready pings database; /jobs APIs query runs/definitions. +- Source connectors store raw docs, DTOs, and mapped canonical advisories with provenance; Update SourceState cursor/backoff. +- Exporters read advisories and write export_state. +## Interfaces & contracts +- IMongoDatabase injected; MongoUrl from options; database name from options or MongoUrl or default "feedser". +- Repositories expose async methods with CancellationToken; deterministic sorting. +- All date/time values stored as UTC; identifiers normalized. +## In/Out of scope +In: persistence, bootstrap, indexes, basic query helpers. +Out: business mapping logic, HTTP, packaging. +## Observability & security expectations +- Log collection/index creation; warn on existing mismatches. +- Timeouts and retry policies; avoid unbounded scans; page reads. +- Do not log DSNs with credentials; redact in diagnostics. +## Tests +- Author and review coverage in `../StellaOps.Feedser.Storage.Mongo.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs new file mode 100644 index 00000000..3614698d --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs @@ -0,0 +1,27 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Advisories; + +[BsonIgnoreExtraElements] +public sealed class AdvisoryDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("advisoryKey")] + public string AdvisoryKey + { + get => Id; + set => Id = value; + } + + [BsonElement("payload")] + public BsonDocument Payload { get; set; } = new(); + + [BsonElement("modified")] + public DateTime Modified { get; set; } + + [BsonElement("published")] + public DateTime? Published { get; set; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs new file mode 100644 index 00000000..3a5ead5a --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs @@ -0,0 +1,245 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Storage.Mongo.Advisories; + +public sealed class AdvisoryStore : IAdvisoryStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public AdvisoryStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.Advisory); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + + public async Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(advisory); + + var payload = CanonicalJsonSerializer.Serialize(advisory); + var document = new AdvisoryDocument + { + AdvisoryKey = advisory.AdvisoryKey, + Payload = BsonDocument.Parse(payload), + Modified = advisory.Modified?.UtcDateTime ?? DateTime.UtcNow, + Published = advisory.Published?.UtcDateTime, + }; + + var options = new ReplaceOptions { IsUpsert = true }; + await _collection.ReplaceOneAsync(x => x.AdvisoryKey == advisory.AdvisoryKey, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted advisory {AdvisoryKey}", advisory.AdvisoryKey); + } + + public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + var document = await _collection.Find(x => x.AdvisoryKey == advisoryKey) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + return document is null ? null : Deserialize(document.Payload); + } + + public async Task> GetRecentAsync(int limit, CancellationToken cancellationToken) + { + var cursor = await _collection.Find(FilterDefinition.Empty) + .SortByDescending(x => x.Modified) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static doc => Deserialize(doc.Payload)).ToArray(); + } + + public async IAsyncEnumerable StreamAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + var options = new FindOptions + { + Sort = Builders.Sort.Ascending(static doc => doc.AdvisoryKey), + }; + + using var cursor = await _collection.FindAsync( + FilterDefinition.Empty, + options, + cancellationToken) + .ConfigureAwait(false); + + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var document in cursor.Current) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return Deserialize(document.Payload); + } + } + } + + private static Advisory Deserialize(BsonDocument payload) + { + ArgumentNullException.ThrowIfNull(payload); + + var advisoryKey = payload.GetValue("advisoryKey", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("advisoryKey missing from payload."); + var title = payload.GetValue("title", defaultValue: null)?.AsString ?? advisoryKey; + + string? summary = payload.TryGetValue("summary", out var summaryValue) && summaryValue.IsString ? summaryValue.AsString : null; + string? language = payload.TryGetValue("language", out var languageValue) && languageValue.IsString ? languageValue.AsString : null; + DateTimeOffset? published = TryReadDateTime(payload, "published"); + DateTimeOffset? modified = TryReadDateTime(payload, "modified"); + string? severity = payload.TryGetValue("severity", out var severityValue) && severityValue.IsString ? severityValue.AsString : null; + var exploitKnown = payload.TryGetValue("exploitKnown", out var exploitValue) && exploitValue.IsBoolean && exploitValue.AsBoolean; + + var aliases = payload.TryGetValue("aliases", out var aliasValue) && aliasValue is BsonArray aliasArray + ? aliasArray.OfType().Where(static x => x.IsString).Select(static x => x.AsString) + : Array.Empty(); + + var references = payload.TryGetValue("references", out var referencesValue) && referencesValue is BsonArray referencesArray + ? referencesArray.OfType().Select(DeserializeReference).ToArray() + : Array.Empty(); + + var affectedPackages = payload.TryGetValue("affectedPackages", out var affectedValue) && affectedValue is BsonArray affectedArray + ? affectedArray.OfType().Select(DeserializeAffectedPackage).ToArray() + : Array.Empty(); + + var cvssMetrics = payload.TryGetValue("cvssMetrics", out var cvssValue) && cvssValue is BsonArray cvssArray + ? cvssArray.OfType().Select(DeserializeCvssMetric).ToArray() + : Array.Empty(); + + var provenance = payload.TryGetValue("provenance", out var provenanceValue) && provenanceValue is BsonArray provenanceArray + ? provenanceArray.OfType().Select(DeserializeProvenance).ToArray() + : Array.Empty(); + + return new Advisory( + advisoryKey, + title, + summary, + language, + published, + modified, + severity, + exploitKnown, + aliases, + references, + affectedPackages, + cvssMetrics, + provenance); + } + + private static AdvisoryReference DeserializeReference(BsonDocument document) + { + var url = document.GetValue("url", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("reference.url missing from payload."); + string? kind = document.TryGetValue("kind", out var kindValue) && kindValue.IsString ? kindValue.AsString : null; + string? sourceTag = document.TryGetValue("sourceTag", out var sourceTagValue) && sourceTagValue.IsString ? sourceTagValue.AsString : null; + string? summary = document.TryGetValue("summary", out var summaryValue) && summaryValue.IsString ? summaryValue.AsString : null; + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + + return new AdvisoryReference(url, kind, sourceTag, summary, provenance); + } + + private static AffectedPackage DeserializeAffectedPackage(BsonDocument document) + { + var type = document.GetValue("type", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("affectedPackages.type missing from payload."); + var identifier = document.GetValue("identifier", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("affectedPackages.identifier missing from payload."); + string? platform = document.TryGetValue("platform", out var platformValue) && platformValue.IsString ? platformValue.AsString : null; + + var versionRanges = document.TryGetValue("versionRanges", out var rangesValue) && rangesValue is BsonArray rangesArray + ? rangesArray.OfType().Select(DeserializeVersionRange).ToArray() + : Array.Empty(); + + var statuses = document.TryGetValue("statuses", out var statusesValue) && statusesValue is BsonArray statusesArray + ? statusesArray.OfType().Select(DeserializeStatus).ToArray() + : Array.Empty(); + + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue is BsonArray provenanceArray + ? provenanceArray.OfType().Select(DeserializeProvenance).ToArray() + : Array.Empty(); + + return new AffectedPackage(type, identifier, platform, versionRanges, statuses, provenance); + } + + private static AffectedVersionRange DeserializeVersionRange(BsonDocument document) + { + var rangeKind = document.GetValue("rangeKind", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("versionRanges.rangeKind missing from payload."); + string? introducedVersion = document.TryGetValue("introducedVersion", out var introducedValue) && introducedValue.IsString ? introducedValue.AsString : null; + string? fixedVersion = document.TryGetValue("fixedVersion", out var fixedValue) && fixedValue.IsString ? fixedValue.AsString : null; + string? lastAffectedVersion = document.TryGetValue("lastAffectedVersion", out var lastValue) && lastValue.IsString ? lastValue.AsString : null; + string? rangeExpression = document.TryGetValue("rangeExpression", out var expressionValue) && expressionValue.IsString ? expressionValue.AsString : null; + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + + return new AffectedVersionRange(rangeKind, introducedVersion, fixedVersion, lastAffectedVersion, rangeExpression, provenance); + } + + private static AffectedPackageStatus DeserializeStatus(BsonDocument document) + { + var status = document.GetValue("status", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("statuses.status missing from payload."); + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + + return new AffectedPackageStatus(status, provenance); + } + + private static CvssMetric DeserializeCvssMetric(BsonDocument document) + { + var version = document.GetValue("version", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("cvssMetrics.version missing from payload."); + var vector = document.GetValue("vector", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("cvssMetrics.vector missing from payload."); + var baseScore = document.TryGetValue("baseScore", out var scoreValue) && scoreValue.IsNumeric ? scoreValue.ToDouble() : 0d; + var baseSeverity = document.GetValue("baseSeverity", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("cvssMetrics.baseSeverity missing from payload."); + var provenance = document.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument + ? DeserializeProvenance(provenanceValue.AsBsonDocument) + : AdvisoryProvenance.Empty; + + return new CvssMetric(version, vector, baseScore, baseSeverity, provenance); + } + + private static AdvisoryProvenance DeserializeProvenance(BsonDocument document) + { + var source = document.GetValue("source", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("provenance.source missing from payload."); + var kind = document.GetValue("kind", defaultValue: null)?.AsString + ?? throw new InvalidOperationException("provenance.kind missing from payload."); + string? value = document.TryGetValue("value", out var valueElement) && valueElement.IsString ? valueElement.AsString : null; + var recordedAt = TryConvertDateTime(document.GetValue("recordedAt", defaultValue: null)); + + return new AdvisoryProvenance(source, kind, value ?? string.Empty, recordedAt ?? DateTimeOffset.UtcNow); + } + + private static DateTimeOffset? TryReadDateTime(BsonDocument document, string field) + => document.TryGetValue(field, out var value) ? TryConvertDateTime(value) : null; + + private static DateTimeOffset? TryConvertDateTime(BsonValue? value) + { + if (value is null) + { + return null; + } + + return value switch + { + BsonDateTime dateTime => DateTime.SpecifyKind(dateTime.ToUniversalTime(), DateTimeKind.Utc), + BsonString stringValue when DateTimeOffset.TryParse(stringValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs new file mode 100644 index 00000000..d1627dd5 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs @@ -0,0 +1,14 @@ +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Storage.Mongo.Advisories; + +public interface IAdvisoryStore +{ + Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken); + + Task FindAsync(string advisoryKey, CancellationToken cancellationToken); + + Task> GetRecentAsync(int limit, CancellationToken cancellationToken); + + IAsyncEnumerable StreamAsync(CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs new file mode 100644 index 00000000..446820b2 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +[BsonIgnoreExtraElements] +public sealed class ChangeHistoryDocument +{ + [BsonId] + public Guid Id { get; set; } + + [BsonElement("source")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("documentId")] + public Guid DocumentId { get; set; } + + [BsonElement("documentSha256")] + public string DocumentSha256 { get; set; } = string.Empty; + + [BsonElement("currentHash")] + public string CurrentHash { get; set; } = string.Empty; + + [BsonElement("previousHash")] + public string? PreviousHash { get; set; } + + [BsonElement("currentSnapshot")] + public string CurrentSnapshot { get; set; } = string.Empty; + + [BsonElement("previousSnapshot")] + public string? PreviousSnapshot { get; set; } + + [BsonElement("changes")] + public List Changes { get; set; } = new(); + + [BsonElement("capturedAt")] + public DateTime CapturedAt { get; set; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs new file mode 100644 index 00000000..3acaf472 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +internal static class ChangeHistoryDocumentExtensions +{ + public static ChangeHistoryDocument ToDocument(this ChangeHistoryRecord record) + { + var changes = new List(record.Changes.Count); + foreach (var change in record.Changes) + { + changes.Add(new BsonDocument + { + ["field"] = change.Field, + ["type"] = change.ChangeType, + ["previous"] = change.PreviousValue is null ? BsonNull.Value : new BsonString(change.PreviousValue), + ["current"] = change.CurrentValue is null ? BsonNull.Value : new BsonString(change.CurrentValue), + }); + } + + return new ChangeHistoryDocument + { + Id = record.Id, + SourceName = record.SourceName, + AdvisoryKey = record.AdvisoryKey, + DocumentId = record.DocumentId, + DocumentSha256 = record.DocumentSha256, + CurrentHash = record.CurrentHash, + PreviousHash = record.PreviousHash, + CurrentSnapshot = record.CurrentSnapshot, + PreviousSnapshot = record.PreviousSnapshot, + Changes = changes, + CapturedAt = record.CapturedAt.UtcDateTime, + }; + } + + public static ChangeHistoryRecord ToRecord(this ChangeHistoryDocument document) + { + var changes = new List(document.Changes.Count); + foreach (var change in document.Changes) + { + var previousValue = change.TryGetValue("previous", out var previousBson) && previousBson is not BsonNull + ? previousBson.AsString + : null; + var currentValue = change.TryGetValue("current", out var currentBson) && currentBson is not BsonNull + ? currentBson.AsString + : null; + var fieldName = change.GetValue("field", "").AsString; + var changeType = change.GetValue("type", "").AsString; + changes.Add(new ChangeHistoryFieldChange(fieldName, changeType, previousValue, currentValue)); + } + + var capturedAtUtc = DateTime.SpecifyKind(document.CapturedAt, DateTimeKind.Utc); + + return new ChangeHistoryRecord( + document.Id, + document.SourceName, + document.AdvisoryKey, + document.DocumentId, + document.DocumentSha256, + document.CurrentHash, + document.PreviousHash, + document.CurrentSnapshot, + document.PreviousSnapshot, + changes, + new DateTimeOffset(capturedAtUtc)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs new file mode 100644 index 00000000..cac29113 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs @@ -0,0 +1,24 @@ +using System; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +public sealed record ChangeHistoryFieldChange +{ + public ChangeHistoryFieldChange(string field, string changeType, string? previousValue, string? currentValue) + { + ArgumentException.ThrowIfNullOrEmpty(field); + ArgumentException.ThrowIfNullOrEmpty(changeType); + Field = field; + ChangeType = changeType; + PreviousValue = previousValue; + CurrentValue = currentValue; + } + + public string Field { get; } + + public string ChangeType { get; } + + public string? PreviousValue { get; } + + public string? CurrentValue { get; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs new file mode 100644 index 00000000..8356ad36 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +public sealed class ChangeHistoryRecord +{ + public ChangeHistoryRecord( + Guid id, + string sourceName, + string advisoryKey, + Guid documentId, + string documentSha256, + string currentHash, + string? previousHash, + string currentSnapshot, + string? previousSnapshot, + IReadOnlyList changes, + DateTimeOffset capturedAt) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + ArgumentException.ThrowIfNullOrEmpty(documentSha256); + ArgumentException.ThrowIfNullOrEmpty(currentHash); + ArgumentException.ThrowIfNullOrEmpty(currentSnapshot); + ArgumentNullException.ThrowIfNull(changes); + + Id = id; + SourceName = sourceName; + AdvisoryKey = advisoryKey; + DocumentId = documentId; + DocumentSha256 = documentSha256; + CurrentHash = currentHash; + PreviousHash = previousHash; + CurrentSnapshot = currentSnapshot; + PreviousSnapshot = previousSnapshot; + Changes = changes; + CapturedAt = capturedAt; + } + + public Guid Id { get; } + + public string SourceName { get; } + + public string AdvisoryKey { get; } + + public Guid DocumentId { get; } + + public string DocumentSha256 { get; } + + public string CurrentHash { get; } + + public string? PreviousHash { get; } + + public string CurrentSnapshot { get; } + + public string? PreviousSnapshot { get; } + + public IReadOnlyList Changes { get; } + + public DateTimeOffset CapturedAt { get; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs new file mode 100644 index 00000000..c1e0df4a --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +public interface IChangeHistoryStore +{ + Task AddAsync(ChangeHistoryRecord record, CancellationToken cancellationToken); + + Task> GetRecentAsync(string sourceName, string advisoryKey, int limit, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs new file mode 100644 index 00000000..8fc9e0ea --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; + +public sealed class MongoChangeHistoryStore : IChangeHistoryStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public MongoChangeHistoryStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.ChangeHistory); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AddAsync(ChangeHistoryRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + var document = record.ToDocument(); + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Recorded change history for {Source}/{Advisory} with hash {Hash}", record.SourceName, record.AdvisoryKey, record.CurrentHash); + } + + public async Task> GetRecentAsync(string sourceName, string advisoryKey, int limit, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + if (limit <= 0) + { + limit = 10; + } + + var cursor = await _collection.Find(x => x.SourceName == sourceName && x.AdvisoryKey == advisoryKey) + .SortByDescending(x => x.CapturedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var records = new List(cursor.Count); + foreach (var document in cursor) + { + records.Add(document.ToRecord()); + } + + return records; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs new file mode 100644 index 00000000..26bf3f4f --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs @@ -0,0 +1,130 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Documents; + +[BsonIgnoreExtraElements] +public sealed class DocumentDocument +{ + [BsonId] + public Guid Id { get; set; } + + [BsonElement("sourceName")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("uri")] + public string Uri { get; set; } = string.Empty; + + [BsonElement("fetchedAt")] + public DateTime FetchedAt { get; set; } + + [BsonElement("sha256")] + public string Sha256 { get; set; } = string.Empty; + + [BsonElement("status")] + public string Status { get; set; } = string.Empty; + + [BsonElement("contentType")] + [BsonIgnoreIfNull] + public string? ContentType { get; set; } + + [BsonElement("headers")] + [BsonIgnoreIfNull] + public BsonDocument? Headers { get; set; } + + [BsonElement("metadata")] + [BsonIgnoreIfNull] + public BsonDocument? Metadata { get; set; } + + [BsonElement("etag")] + [BsonIgnoreIfNull] + public string? Etag { get; set; } + + [BsonElement("lastModified")] + [BsonIgnoreIfNull] + public DateTime? LastModified { get; set; } + + [BsonElement("expiresAt")] + [BsonIgnoreIfNull] + public DateTime? ExpiresAt { get; set; } + + [BsonElement("gridFsId")] + [BsonIgnoreIfNull] + public ObjectId? GridFsId { get; set; } +} + +internal static class DocumentDocumentExtensions +{ + public static DocumentDocument FromRecord(DocumentRecord record) + { + return new DocumentDocument + { + Id = record.Id, + SourceName = record.SourceName, + Uri = record.Uri, + FetchedAt = record.FetchedAt.UtcDateTime, + Sha256 = record.Sha256, + Status = record.Status, + ContentType = record.ContentType, + Headers = ToBson(record.Headers), + Metadata = ToBson(record.Metadata), + Etag = record.Etag, + LastModified = record.LastModified?.UtcDateTime, + GridFsId = record.GridFsId, + ExpiresAt = record.ExpiresAt?.UtcDateTime, + }; + } + + public static DocumentRecord ToRecord(this DocumentDocument document) + { + IReadOnlyDictionary? headers = null; + if (document.Headers is not null) + { + headers = document.Headers.Elements.ToDictionary( + static e => e.Name, + static e => e.Value?.ToString() ?? string.Empty, + StringComparer.Ordinal); + } + + IReadOnlyDictionary? metadata = null; + if (document.Metadata is not null) + { + metadata = document.Metadata.Elements.ToDictionary( + static e => e.Name, + static e => e.Value?.ToString() ?? string.Empty, + StringComparer.Ordinal); + } + + return new DocumentRecord( + document.Id, + document.SourceName, + document.Uri, + DateTime.SpecifyKind(document.FetchedAt, DateTimeKind.Utc), + document.Sha256, + document.Status, + document.ContentType, + headers, + metadata, + document.Etag, + document.LastModified.HasValue ? DateTime.SpecifyKind(document.LastModified.Value, DateTimeKind.Utc) : null, + document.GridFsId, + document.ExpiresAt.HasValue ? DateTime.SpecifyKind(document.ExpiresAt.Value, DateTimeKind.Utc) : null); + } + + private static BsonDocument? ToBson(IReadOnlyDictionary? values) + { + if (values is null) + { + return null; + } + + var document = new BsonDocument(); + foreach (var kvp in values) + { + document[kvp.Key] = kvp.Value; + } + + return document; + } + +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs new file mode 100644 index 00000000..9a81851c --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs @@ -0,0 +1,22 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo.Documents; + +public sealed record DocumentRecord( + Guid Id, + string SourceName, + string Uri, + DateTimeOffset FetchedAt, + string Sha256, + string Status, + string? ContentType, + IReadOnlyDictionary? Headers, + IReadOnlyDictionary? Metadata, + string? Etag, + DateTimeOffset? LastModified, + ObjectId? GridFsId, + DateTimeOffset? ExpiresAt = null) +{ + public DocumentRecord WithStatus(string status) + => this with { Status = status }; +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs new file mode 100644 index 00000000..8ae37335 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Documents; + +public sealed class DocumentStore : IDocumentStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public DocumentStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.Document); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + var document = DocumentDocumentExtensions.FromRecord(record); + var filter = Builders.Filter.Eq(x => x.SourceName, record.SourceName) + & Builders.Filter.Eq(x => x.Uri, record.Uri); + + var options = new FindOneAndReplaceOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After, + }; + + var replaced = await _collection.FindOneAndReplaceAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted document {Source}/{Uri}", record.SourceName, record.Uri); + return (replaced ?? document).ToRecord(); + } + + public async Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(uri); + + var filter = Builders.Filter.Eq(x => x.SourceName, sourceName) + & Builders.Filter.Eq(x => x.Uri, uri); + + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task FindAsync(Guid id, CancellationToken cancellationToken) + { + var document = await _collection.Find(x => x.Id == id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(status); + + var update = Builders.Update + .Set(x => x.Status, status) + .Set(x => x.LastModified, DateTime.UtcNow); + + var result = await _collection.UpdateOneAsync(x => x.Id == id, update, cancellationToken: cancellationToken).ConfigureAwait(false); + return result.MatchedCount > 0; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs new file mode 100644 index 00000000..1d7940b3 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.Storage.Mongo.Documents; + +public interface IDocumentStore +{ + Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken); + + Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken); + + Task FindAsync(Guid id, CancellationToken cancellationToken); + + Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs new file mode 100644 index 00000000..f1317c08 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs @@ -0,0 +1,49 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Dtos; + +[BsonIgnoreExtraElements] +public sealed class DtoDocument +{ + [BsonId] + public Guid Id { get; set; } + + [BsonElement("documentId")] + public Guid DocumentId { get; set; } + + [BsonElement("sourceName")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("schemaVersion")] + public string SchemaVersion { get; set; } = string.Empty; + + [BsonElement("payload")] + public BsonDocument Payload { get; set; } = new(); + + [BsonElement("validatedAt")] + public DateTime ValidatedAt { get; set; } +} + +internal static class DtoDocumentExtensions +{ + public static DtoDocument FromRecord(DtoRecord record) + => new() + { + Id = record.Id, + DocumentId = record.DocumentId, + SourceName = record.SourceName, + SchemaVersion = record.SchemaVersion, + Payload = record.Payload ?? new BsonDocument(), + ValidatedAt = record.ValidatedAt.UtcDateTime, + }; + + public static DtoRecord ToRecord(this DtoDocument document) + => new( + document.Id, + document.DocumentId, + document.SourceName, + document.SchemaVersion, + document.Payload, + DateTime.SpecifyKind(document.ValidatedAt, DateTimeKind.Utc)); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs new file mode 100644 index 00000000..e76a2b69 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs @@ -0,0 +1,11 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo.Dtos; + +public sealed record DtoRecord( + Guid Id, + Guid DocumentId, + string SourceName, + string SchemaVersion, + BsonDocument Payload, + DateTimeOffset ValidatedAt); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs new file mode 100644 index 00000000..1d7c02ff --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs @@ -0,0 +1,55 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Dtos; + +public sealed class DtoStore : IDtoStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public DtoStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.Dto); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task UpsertAsync(DtoRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + var document = DtoDocumentExtensions.FromRecord(record); + var filter = Builders.Filter.Eq(x => x.DocumentId, record.DocumentId) + & Builders.Filter.Eq(x => x.SourceName, record.SourceName); + + var options = new FindOneAndReplaceOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After, + }; + + var replaced = await _collection.FindOneAndReplaceAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted DTO for {Source}/{DocumentId}", record.SourceName, record.DocumentId); + return (replaced ?? document).ToRecord(); + } + + public async Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken) + { + var document = await _collection.Find(x => x.DocumentId == documentId) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken) + { + var cursor = await _collection.Find(x => x.SourceName == sourceName) + .SortByDescending(x => x.ValidatedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static x => x.ToRecord()).ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs new file mode 100644 index 00000000..b5eee6c0 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Feedser.Storage.Mongo.Dtos; + +public interface IDtoStore +{ + Task UpsertAsync(DtoRecord record, CancellationToken cancellationToken); + + Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken); + + Task> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs new file mode 100644 index 00000000..eb83b1cd --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs @@ -0,0 +1,63 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +[BsonIgnoreExtraElements] +public sealed class ExportStateDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("baseExportId")] + public string? BaseExportId { get; set; } + + [BsonElement("baseDigest")] + public string? BaseDigest { get; set; } + + [BsonElement("lastFullDigest")] + public string? LastFullDigest { get; set; } + + [BsonElement("lastDeltaDigest")] + public string? LastDeltaDigest { get; set; } + + [BsonElement("exportCursor")] + public string? ExportCursor { get; set; } + + [BsonElement("targetRepo")] + public string? TargetRepository { get; set; } + + [BsonElement("exporterVersion")] + public string? ExporterVersion { get; set; } + + [BsonElement("updatedAt")] + public DateTime UpdatedAt { get; set; } +} + +internal static class ExportStateDocumentExtensions +{ + public static ExportStateDocument FromRecord(ExportStateRecord record) + => new() + { + Id = record.Id, + BaseExportId = record.BaseExportId, + BaseDigest = record.BaseDigest, + LastFullDigest = record.LastFullDigest, + LastDeltaDigest = record.LastDeltaDigest, + ExportCursor = record.ExportCursor, + TargetRepository = record.TargetRepository, + ExporterVersion = record.ExporterVersion, + UpdatedAt = record.UpdatedAt.UtcDateTime, + }; + + public static ExportStateRecord ToRecord(this ExportStateDocument document) + => new( + document.Id, + document.BaseExportId, + document.BaseDigest, + document.LastFullDigest, + document.LastDeltaDigest, + document.ExportCursor, + document.TargetRepository, + document.ExporterVersion, + DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc)); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs new file mode 100644 index 00000000..c1c09acf --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs @@ -0,0 +1,102 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +/// +/// Helper for exporters to read and persist their export metadata in Mongo-backed storage. +/// +public sealed class ExportStateManager +{ + private readonly IExportStateStore _store; + private readonly TimeProvider _timeProvider; + + public ExportStateManager(IExportStateStore store, TimeProvider? timeProvider = null) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public Task GetAsync(string exporterId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(exporterId); + return _store.FindAsync(exporterId, cancellationToken); + } + + public async Task StoreFullExportAsync( + string exporterId, + string exportId, + string exportDigest, + string? cursor, + string? targetRepository, + string exporterVersion, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(exporterId); + ArgumentException.ThrowIfNullOrEmpty(exportId); + ArgumentException.ThrowIfNullOrEmpty(exportDigest); + ArgumentException.ThrowIfNullOrEmpty(exporterVersion); + + var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false); + var repository = string.IsNullOrWhiteSpace(targetRepository) ? existing?.TargetRepository : targetRepository; + var now = _timeProvider.GetUtcNow(); + + var baseExportId = existing?.BaseExportId ?? exportId; + var baseDigest = existing?.BaseDigest ?? exportDigest; + + var record = existing is null + ? new ExportStateRecord( + exporterId, + baseExportId, + baseDigest, + exportDigest, + LastDeltaDigest: null, + ExportCursor: cursor ?? exportDigest, + TargetRepository: repository, + ExporterVersion: exporterVersion, + UpdatedAt: now) + : existing with + { + BaseExportId = baseExportId, + BaseDigest = baseDigest, + LastFullDigest = exportDigest, + LastDeltaDigest = null, + ExportCursor = cursor ?? existing.ExportCursor, + TargetRepository = repository, + ExporterVersion = exporterVersion, + UpdatedAt = now, + }; + + return await _store.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + } + + public async Task StoreDeltaExportAsync( + string exporterId, + string deltaDigest, + string? cursor, + string exporterVersion, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(exporterId); + ArgumentException.ThrowIfNullOrEmpty(deltaDigest); + ArgumentException.ThrowIfNullOrEmpty(exporterVersion); + + var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false); + if (existing is null) + { + throw new InvalidOperationException($"Full export state missing for '{exporterId}'."); + } + + var now = _timeProvider.GetUtcNow(); + var record = existing with + { + LastDeltaDigest = deltaDigest, + ExportCursor = cursor ?? existing.ExportCursor, + ExporterVersion = exporterVersion, + UpdatedAt = now, + }; + + return await _store.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs new file mode 100644 index 00000000..a5597e4e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +public sealed record ExportStateRecord( + string Id, + string? BaseExportId, + string? BaseDigest, + string? LastFullDigest, + string? LastDeltaDigest, + string? ExportCursor, + string? TargetRepository, + string? ExporterVersion, + DateTimeOffset UpdatedAt); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs new file mode 100644 index 00000000..8d34e573 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +public sealed class ExportStateStore : IExportStateStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public ExportStateStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.ExportState); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + var document = ExportStateDocumentExtensions.FromRecord(record); + var options = new FindOneAndReplaceOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After, + }; + + var replaced = await _collection.FindOneAndReplaceAsync( + x => x.Id == record.Id, + document, + options, + cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Stored export state {StateId}", record.Id); + return (replaced ?? document).ToRecord(); + } + + public async Task FindAsync(string id, CancellationToken cancellationToken) + { + var document = await _collection.Find(x => x.Id == id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs new file mode 100644 index 00000000..a331dffb --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Storage.Mongo.Exporting; + +public interface IExportStateStore +{ + Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken); + + Task FindAsync(string id, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs b/src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs new file mode 100644 index 00000000..5887a631 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs @@ -0,0 +1,14 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo; + +public interface ISourceStateRepository +{ + Task TryGetAsync(string sourceName, CancellationToken cancellationToken); + + Task UpsertAsync(SourceStateRecord record, CancellationToken cancellationToken); + + Task UpdateCursorAsync(string sourceName, BsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken); + + Task MarkFailureAsync(string sourceName, DateTimeOffset failedAt, TimeSpan? backoff, string? failureReason, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs new file mode 100644 index 00000000..e88565b1 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs @@ -0,0 +1,38 @@ +using MongoDB.Bson.Serialization.Attributes; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Storage.Mongo; + +[BsonIgnoreExtraElements] +public sealed class JobLeaseDocument +{ + [BsonId] + public string Key { get; set; } = string.Empty; + + [BsonElement("holder")] + public string Holder { get; set; } = string.Empty; + + [BsonElement("acquiredAt")] + public DateTime AcquiredAt { get; set; } + + [BsonElement("heartbeatAt")] + public DateTime HeartbeatAt { get; set; } + + [BsonElement("leaseMs")] + public long LeaseMs { get; set; } + + [BsonElement("ttlAt")] + public DateTime TtlAt { get; set; } +} + +internal static class JobLeaseDocumentExtensions +{ + public static JobLease ToLease(this JobLeaseDocument document) + => new( + document.Key, + document.Holder, + DateTime.SpecifyKind(document.AcquiredAt, DateTimeKind.Utc), + DateTime.SpecifyKind(document.HeartbeatAt, DateTimeKind.Utc), + TimeSpan.FromMilliseconds(document.LeaseMs), + DateTime.SpecifyKind(document.TtlAt, DateTimeKind.Utc)); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs new file mode 100644 index 00000000..3cf6aa75 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs @@ -0,0 +1,119 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Storage.Mongo; + +[BsonIgnoreExtraElements] +public sealed class JobRunDocument +{ + [BsonId] + [BsonGuidRepresentation(GuidRepresentation.Standard)] + public Guid Id { get; set; } + + [BsonElement("kind")] + public string Kind { get; set; } = string.Empty; + + [BsonElement("status")] + public string Status { get; set; } = JobRunStatus.Pending.ToString(); + + [BsonElement("trigger")] + public string Trigger { get; set; } = string.Empty; + + [BsonElement("parameters")] + public BsonDocument Parameters { get; set; } = new(); + + [BsonElement("parametersHash")] + [BsonIgnoreIfNull] + public string? ParametersHash { get; set; } + + [BsonElement("createdAt")] + public DateTime CreatedAt { get; set; } + + [BsonElement("startedAt")] + [BsonIgnoreIfNull] + public DateTime? StartedAt { get; set; } + + [BsonElement("completedAt")] + [BsonIgnoreIfNull] + public DateTime? CompletedAt { get; set; } + + [BsonElement("error")] + [BsonIgnoreIfNull] + public string? Error { get; set; } + + [BsonElement("timeoutMs")] + [BsonIgnoreIfNull] + public long? TimeoutMs { get; set; } + + [BsonElement("leaseMs")] + [BsonIgnoreIfNull] + public long? LeaseMs { get; set; } +} + +internal static class JobRunDocumentExtensions +{ + public static JobRunDocument FromRequest(JobRunCreateRequest request, Guid id) + { + return new JobRunDocument + { + Id = id, + Kind = request.Kind, + Status = JobRunStatus.Pending.ToString(), + Trigger = request.Trigger, + Parameters = request.Parameters is { Count: > 0 } + ? BsonDocument.Parse(JsonSerializer.Serialize(request.Parameters)) + : new BsonDocument(), + ParametersHash = request.ParametersHash, + CreatedAt = request.CreatedAt.UtcDateTime, + TimeoutMs = request.Timeout?.MillisecondsFromTimespan(), + LeaseMs = request.LeaseDuration?.MillisecondsFromTimespan(), + }; + } + + public static JobRunSnapshot ToSnapshot(this JobRunDocument document) + { + var parameters = document.Parameters?.ToDictionary() ?? new Dictionary(); + + return new JobRunSnapshot( + document.Id, + document.Kind, + Enum.Parse(document.Status, ignoreCase: true), + DateTime.SpecifyKind(document.CreatedAt, DateTimeKind.Utc), + document.StartedAt.HasValue ? DateTime.SpecifyKind(document.StartedAt.Value, DateTimeKind.Utc) : null, + document.CompletedAt.HasValue ? DateTime.SpecifyKind(document.CompletedAt.Value, DateTimeKind.Utc) : null, + document.Trigger, + document.ParametersHash, + document.Error, + document.TimeoutMs?.MillisecondsToTimespan(), + document.LeaseMs?.MillisecondsToTimespan(), + parameters); + } + + public static Dictionary ToDictionary(this BsonDocument document) + { + return document.Elements.ToDictionary( + static element => element.Name, + static element => element.Value switch + { + BsonString s => (object?)s.AsString, + BsonBoolean b => b.AsBoolean, + BsonInt32 i => i.AsInt32, + BsonInt64 l => l.AsInt64, + BsonDouble d => d.AsDouble, + BsonNull => null, + BsonArray array => array.Select(v => v.IsBsonDocument ? ToDictionary(v.AsBsonDocument) : (object?)v.ToString()).ToArray(), + BsonDocument doc => ToDictionary(doc), + _ => element.Value.ToString(), + }); + } + + private static long MillisecondsFromTimespan(this TimeSpan timeSpan) + => (long)timeSpan.TotalMilliseconds; + + private static TimeSpan MillisecondsToTimespan(this long milliseconds) + => TimeSpan.FromMilliseconds(milliseconds); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs new file mode 100644 index 00000000..3f58bd10 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.JpFlags; + +public interface IJpFlagStore +{ + Task UpsertAsync(JpFlagRecord record, CancellationToken cancellationToken); + + Task FindAsync(string advisoryKey, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs new file mode 100644 index 00000000..d7640ed8 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs @@ -0,0 +1,54 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.JpFlags; + +[BsonIgnoreExtraElements] +public sealed class JpFlagDocument +{ + [BsonId] + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("sourceName")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("category")] + [BsonIgnoreIfNull] + public string? Category { get; set; } + + [BsonElement("vendorStatus")] + [BsonIgnoreIfNull] + public string? VendorStatus { get; set; } + + [BsonElement("recordedAt")] + public DateTime RecordedAt { get; set; } +} + +internal static class JpFlagDocumentExtensions +{ + public static JpFlagDocument FromRecord(JpFlagRecord record) + { + ArgumentNullException.ThrowIfNull(record); + + return new JpFlagDocument + { + AdvisoryKey = record.AdvisoryKey, + SourceName = record.SourceName, + Category = record.Category, + VendorStatus = record.VendorStatus, + RecordedAt = record.RecordedAt.UtcDateTime, + }; + } + + public static JpFlagRecord ToRecord(this JpFlagDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + return new JpFlagRecord( + document.AdvisoryKey, + document.SourceName, + document.Category, + document.VendorStatus, + DateTime.SpecifyKind(document.RecordedAt, DateTimeKind.Utc)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs new file mode 100644 index 00000000..88cc8e52 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Feedser.Storage.Mongo.JpFlags; + +/// +/// Captures Japan-specific enrichment flags derived from JVN payloads. +/// +public sealed record JpFlagRecord( + string AdvisoryKey, + string SourceName, + string? Category, + string? VendorStatus, + DateTimeOffset RecordedAt) +{ + public JpFlagRecord WithRecordedAt(DateTimeOffset recordedAt) + => this with { RecordedAt = recordedAt.ToUniversalTime() }; +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs new file mode 100644 index 00000000..b5b62ccc --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.JpFlags; + +public sealed class JpFlagStore : IJpFlagStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public JpFlagStore(IMongoDatabase database, ILogger logger) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(logger); + + _collection = database.GetCollection(MongoStorageDefaults.Collections.JpFlags); + _logger = logger; + } + + public async Task UpsertAsync(JpFlagRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + + var document = JpFlagDocumentExtensions.FromRecord(record); + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, record.AdvisoryKey); + var options = new ReplaceOptions { IsUpsert = true }; + await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted jp_flag for {AdvisoryKey}", record.AdvisoryKey); + } + + public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md b/src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md new file mode 100644 index 00000000..8423e22d --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md @@ -0,0 +1,37 @@ +# Mongo Schema Migration Playbook + +This module owns the persistent shape of Feedser's MongoDB database. Upgrades must be deterministic and safe to run on live replicas. The `MongoMigrationRunner` executes idempotent migrations on startup immediately after the bootstrapper completes its collection and index checks. + +## Execution Path + +1. `StellaOps.Feedser.WebService` calls `MongoBootstrapper.InitializeAsync()` during startup. +2. Once collections and baseline indexes are ensured, the bootstrapper invokes `MongoMigrationRunner.RunAsync()`. +3. Each `IMongoMigration` implementation is sorted by its `Id` (ordinal compare) and executed exactly once. Completion is recorded in the `schema_migrations` collection. +4. Failures surface during startup and prevent the service from serving traffic, matching our "fail-fast" requirement for storage incompatibilities. + +## Creating a Migration + +1. Implement `IMongoMigration` under `StellaOps.Feedser.Storage.Mongo.Migrations`. Use a monotonically increasing identifier such as `yyyyMMdd_description`. +2. Keep the body idempotent: query state first, drop/re-create indexes only when mismatch is detected, and avoid multi-document transactions unless required. +3. Add the migration to DI in `ServiceCollectionExtensions` so it flows into the runner. +4. Write an integration test that exercises the migration against a Mongo2Go instance to validate behaviour. + +## Current Migrations + +| Id | Description | +| --- | --- | +| `20241005_document_expiry_indexes` | Ensures `document` collection uses the correct TTL/partial index depending on raw document retention settings. | +| `20241005_gridfs_expiry_indexes` | Aligns the GridFS `documents.files` TTL index with retention settings. | + +## Operator Runbook + +- `schema_migrations` records each applied migration (`_id`, `description`, `appliedAt`). Review this collection when auditing upgrades. +- To re-run a migration in a lab, delete the corresponding document from `schema_migrations` and restart the service. **Do not** do this in production unless the migration body is known to be idempotent and safe. +- When changing retention settings (`RawDocumentRetention`), deploy the new configuration and restart Feedser. The migration runner will adjust indexes on the next boot. +- If migrations fail, restart with `Logging__LogLevel__StellaOps.Feedser.Storage.Mongo.Migrations=Debug` to surface diagnostic output. Remediate underlying index/collection drift before retrying. + +## Validating an Upgrade + +1. Run `dotnet test --filter MongoMigrationRunnerTests` to exercise integration coverage. +2. In staging, execute `db.schema_migrations.find().sort({_id:1})` to verify applied migrations and timestamps. +3. Inspect index shapes: `db.document.getIndexes()` and `db.documents.files.getIndexes()` for TTL/partial filter alignment. diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs new file mode 100644 index 00000000..f8179dad --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +public interface IMergeEventStore +{ + Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken); + + Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs new file mode 100644 index 00000000..ed6466f9 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs @@ -0,0 +1,48 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +[BsonIgnoreExtraElements] +public sealed class MergeEventDocument +{ + [BsonId] + public Guid Id { get; set; } + + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("beforeHash")] + public byte[] BeforeHash { get; set; } = Array.Empty(); + + [BsonElement("afterHash")] + public byte[] AfterHash { get; set; } = Array.Empty(); + + [BsonElement("mergedAt")] + public DateTime MergedAt { get; set; } + + [BsonElement("inputDocuments")] + public List InputDocuments { get; set; } = new(); +} + +internal static class MergeEventDocumentExtensions +{ + public static MergeEventDocument FromRecord(MergeEventRecord record) + => new() + { + Id = record.Id, + AdvisoryKey = record.AdvisoryKey, + BeforeHash = record.BeforeHash, + AfterHash = record.AfterHash, + MergedAt = record.MergedAt.UtcDateTime, + InputDocuments = record.InputDocumentIds.ToList(), + }; + + public static MergeEventRecord ToRecord(this MergeEventDocument document) + => new( + document.Id, + document.AdvisoryKey, + document.BeforeHash, + document.AfterHash, + DateTime.SpecifyKind(document.MergedAt, DateTimeKind.Utc), + document.InputDocuments); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs new file mode 100644 index 00000000..83ce8afd --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +public sealed record MergeEventRecord( + Guid Id, + string AdvisoryKey, + byte[] BeforeHash, + byte[] AfterHash, + DateTimeOffset MergedAt, + IReadOnlyList InputDocumentIds); diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs new file mode 100644 index 00000000..df30112e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs @@ -0,0 +1,36 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; + +public sealed class MergeEventStore : IMergeEventStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public MergeEventStore(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.MergeEvent); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + var document = MergeEventDocumentExtensions.FromRecord(record); + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Appended merge event {MergeId} for {AdvisoryKey}", record.Id, record.AdvisoryKey); + } + + public async Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken) + { + var cursor = await _collection.Find(x => x.AdvisoryKey == advisoryKey) + .SortByDescending(x => x.MergedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static x => x.ToRecord()).ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs new file mode 100644 index 00000000..5dd052df --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs @@ -0,0 +1,146 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +internal sealed class EnsureDocumentExpiryIndexesMigration : IMongoMigration +{ + private readonly MongoStorageOptions _options; + + public EnsureDocumentExpiryIndexesMigration(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + _options = options.Value; + } + + public string Id => "20241005_document_expiry_indexes"; + + public string Description => "Ensure document.expiresAt index matches configured retention"; + + public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + var needsTtl = _options.RawDocumentRetention > TimeSpan.Zero; + var collection = database.GetCollection(MongoStorageDefaults.Collections.Document); + + using var cursor = await collection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false); + var indexes = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + + var ttlIndex = indexes.FirstOrDefault(x => TryGetName(x, out var name) && string.Equals(name, "document_expiresAt_ttl", StringComparison.Ordinal)); + var nonTtlIndex = indexes.FirstOrDefault(x => TryGetName(x, out var name) && string.Equals(name, "document_expiresAt", StringComparison.Ordinal)); + + if (needsTtl) + { + var shouldRebuild = ttlIndex is null || !IndexMatchesTtlExpectations(ttlIndex); + if (shouldRebuild) + { + if (ttlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt_ttl", cancellationToken).ConfigureAwait(false); + } + + if (nonTtlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt", cancellationToken).ConfigureAwait(false); + } + + var options = new CreateIndexOptions + { + Name = "document_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("expiresAt", true), + }; + + var keys = Builders.IndexKeys.Ascending("expiresAt"); + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options), cancellationToken: cancellationToken).ConfigureAwait(false); + } + else if (nonTtlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt", cancellationToken).ConfigureAwait(false); + } + } + else + { + if (ttlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt_ttl", cancellationToken).ConfigureAwait(false); + } + + var shouldRebuild = nonTtlIndex is null || !IndexMatchesNonTtlExpectations(nonTtlIndex); + if (shouldRebuild) + { + if (nonTtlIndex is not null) + { + await collection.Indexes.DropOneAsync("document_expiresAt", cancellationToken).ConfigureAwait(false); + } + + var options = new CreateIndexOptions + { + Name = "document_expiresAt", + PartialFilterExpression = Builders.Filter.Exists("expiresAt", true), + }; + + var keys = Builders.IndexKeys.Ascending("expiresAt"); + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options), cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + } + + private static bool IndexMatchesTtlExpectations(BsonDocument index) + { + if (!index.TryGetValue("expireAfterSeconds", out var expireAfter) || expireAfter.ToDouble() != 0) + { + return false; + } + + if (!index.TryGetValue("partialFilterExpression", out var partialFilter) || partialFilter is not BsonDocument partialDoc) + { + return false; + } + + if (!partialDoc.TryGetValue("expiresAt", out var expiresAtRule) || expiresAtRule is not BsonDocument expiresAtDoc) + { + return false; + } + + return expiresAtDoc.Contains("$exists") && expiresAtDoc["$exists"].ToBoolean(); + } + + private static bool IndexMatchesNonTtlExpectations(BsonDocument index) + { + if (index.Contains("expireAfterSeconds")) + { + return false; + } + + if (!index.TryGetValue("partialFilterExpression", out var partialFilter) || partialFilter is not BsonDocument partialDoc) + { + return false; + } + + if (!partialDoc.TryGetValue("expiresAt", out var expiresAtRule) || expiresAtRule is not BsonDocument expiresAtDoc) + { + return false; + } + + return expiresAtDoc.Contains("$exists") && expiresAtDoc["$exists"].ToBoolean(); + } + + private static bool TryGetName(BsonDocument index, out string name) + { + if (index.TryGetValue("name", out var value) && value.IsString) + { + name = value.AsString; + return true; + } + + name = string.Empty; + return false; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs new file mode 100644 index 00000000..7f15de37 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs @@ -0,0 +1,95 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +internal sealed class EnsureGridFsExpiryIndexesMigration : IMongoMigration +{ + private readonly MongoStorageOptions _options; + + public EnsureGridFsExpiryIndexesMigration(IOptions options) + { + ArgumentNullException.ThrowIfNull(options); + _options = options.Value; + } + + public string Id => "20241005_gridfs_expiry_indexes"; + + public string Description => "Ensure GridFS metadata.expiresAt TTL index reflects retention settings"; + + public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + var needsTtl = _options.RawDocumentRetention > TimeSpan.Zero; + var collection = database.GetCollection("documents.files"); + + using var cursor = await collection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false); + var indexes = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + + var ttlIndex = indexes.FirstOrDefault(x => TryGetName(x, out var name) && string.Equals(name, "gridfs_files_expiresAt_ttl", StringComparison.Ordinal)); + + if (needsTtl) + { + var shouldRebuild = ttlIndex is null || !IndexMatchesTtlExpectations(ttlIndex); + if (shouldRebuild) + { + if (ttlIndex is not null) + { + await collection.Indexes.DropOneAsync("gridfs_files_expiresAt_ttl", cancellationToken).ConfigureAwait(false); + } + + var keys = Builders.IndexKeys.Ascending("metadata.expiresAt"); + var options = new CreateIndexOptions + { + Name = "gridfs_files_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("metadata.expiresAt", true), + }; + + await collection.Indexes.CreateOneAsync(new CreateIndexModel(keys, options), cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + else if (ttlIndex is not null) + { + await collection.Indexes.DropOneAsync("gridfs_files_expiresAt_ttl", cancellationToken).ConfigureAwait(false); + } + } + + private static bool IndexMatchesTtlExpectations(BsonDocument index) + { + if (!index.TryGetValue("expireAfterSeconds", out var expireAfter) || expireAfter.ToDouble() != 0) + { + return false; + } + + if (!index.TryGetValue("partialFilterExpression", out var partialFilter) || partialFilter is not BsonDocument partialDoc) + { + return false; + } + + if (!partialDoc.TryGetValue("metadata.expiresAt", out var expiresAtRule) || expiresAtRule is not BsonDocument expiresAtDoc) + { + return false; + } + + return expiresAtDoc.Contains("$exists") && expiresAtDoc["$exists"].ToBoolean(); + } + + private static bool TryGetName(BsonDocument index, out string name) + { + if (index.TryGetValue("name", out var value) && value.IsString) + { + name = value.AsString; + return true; + } + + name = string.Empty; + return false; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs new file mode 100644 index 00000000..84b1193c --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs @@ -0,0 +1,24 @@ +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +/// +/// Represents a single, idempotent MongoDB migration. +/// +public interface IMongoMigration +{ + /// + /// Unique identifier for the migration. Sorting is performed using ordinal comparison. + /// + string Id { get; } + + /// + /// Short description surfaced in logs to aid runbooks. + /// + string Description { get; } + + /// + /// Executes the migration. + /// + Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs new file mode 100644 index 00000000..0ef5db7a --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs @@ -0,0 +1,18 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +[BsonIgnoreExtraElements] +internal sealed class MongoMigrationDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("description")] + [BsonIgnoreIfNull] + public string? Description { get; set; } + + [BsonElement("appliedAt")] + public DateTime AppliedAtUtc { get; set; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs b/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs new file mode 100644 index 00000000..04c52745 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.Migrations; + +/// +/// Executes pending schema migrations tracked inside MongoDB to keep upgrades deterministic. +/// +public sealed class MongoMigrationRunner +{ + private readonly IMongoDatabase _database; + private readonly IReadOnlyList _migrations; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public MongoMigrationRunner( + IMongoDatabase database, + IEnumerable migrations, + ILogger logger, + TimeProvider? timeProvider = null) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _migrations = (migrations ?? throw new ArgumentNullException(nameof(migrations))) + .OrderBy(m => m.Id, StringComparer.Ordinal) + .ToArray(); + } + + public async Task RunAsync(CancellationToken cancellationToken) + { + if (_migrations.Count == 0) + { + return; + } + + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Migrations); + await EnsureCollectionExistsAsync(_database, cancellationToken).ConfigureAwait(false); + + var appliedIds = await LoadAppliedMigrationIdsAsync(collection, cancellationToken).ConfigureAwait(false); + foreach (var migration in _migrations) + { + if (appliedIds.Contains(migration.Id, StringComparer.Ordinal)) + { + continue; + } + + _logger.LogInformation("Applying Mongo migration {MigrationId}: {Description}", migration.Id, migration.Description); + try + { + await migration.ApplyAsync(_database, cancellationToken).ConfigureAwait(false); + var document = new MongoMigrationDocument + { + Id = migration.Id, + Description = string.IsNullOrWhiteSpace(migration.Description) ? null : migration.Description, + AppliedAtUtc = _timeProvider.GetUtcNow().UtcDateTime, + }; + + await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Mongo migration {MigrationId} applied", migration.Id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Mongo migration {MigrationId} failed", migration.Id); + throw; + } + } + } + + private static async Task> LoadAppliedMigrationIdsAsync( + IMongoCollection collection, + CancellationToken cancellationToken) + { + using var cursor = await collection.FindAsync(FilterDefinition.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); + var applied = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + var set = new HashSet(StringComparer.Ordinal); + foreach (var document in applied) + { + if (!string.IsNullOrWhiteSpace(document.Id)) + { + set.Add(document.Id); + } + } + + return set; + } + + private static async Task EnsureCollectionExistsAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + using var cursor = await database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var names = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + if (!names.Contains(MongoStorageDefaults.Collections.Migrations, StringComparer.Ordinal)) + { + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs new file mode 100644 index 00000000..9ea86565 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs @@ -0,0 +1,306 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Storage.Mongo.Migrations; + +namespace StellaOps.Feedser.Storage.Mongo; + +/// +/// Ensures required collections and indexes exist before the service begins processing. +/// +public sealed class MongoBootstrapper +{ + private const string RawDocumentBucketName = "documents"; + private static readonly string[] RequiredCollections = + { + MongoStorageDefaults.Collections.Source, + MongoStorageDefaults.Collections.SourceState, + MongoStorageDefaults.Collections.Document, + MongoStorageDefaults.Collections.Dto, + MongoStorageDefaults.Collections.Advisory, + MongoStorageDefaults.Collections.Alias, + MongoStorageDefaults.Collections.Affected, + MongoStorageDefaults.Collections.Reference, + MongoStorageDefaults.Collections.KevFlag, + MongoStorageDefaults.Collections.RuFlags, + MongoStorageDefaults.Collections.JpFlags, + MongoStorageDefaults.Collections.PsirtFlags, + MongoStorageDefaults.Collections.MergeEvent, + MongoStorageDefaults.Collections.ExportState, + MongoStorageDefaults.Collections.ChangeHistory, + MongoStorageDefaults.Collections.Locks, + MongoStorageDefaults.Collections.Jobs, + MongoStorageDefaults.Collections.Migrations, + }; + + private readonly IMongoDatabase _database; + private readonly MongoStorageOptions _options; + private readonly ILogger _logger; + private readonly MongoMigrationRunner _migrationRunner; + + public MongoBootstrapper( + IMongoDatabase database, + IOptions options, + ILogger logger, + MongoMigrationRunner migrationRunner) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _migrationRunner = migrationRunner ?? throw new ArgumentNullException(nameof(migrationRunner)); + } + + public async Task InitializeAsync(CancellationToken cancellationToken) + { + var existingCollections = await ListCollectionsAsync(cancellationToken).ConfigureAwait(false); + + foreach (var collectionName in RequiredCollections) + { + if (!existingCollections.Contains(collectionName)) + { + await _database.CreateCollectionAsync(collectionName, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Created Mongo collection {Collection}", collectionName); + } + } + + await Task.WhenAll( + EnsureLocksIndexesAsync(cancellationToken), + EnsureJobsIndexesAsync(cancellationToken), + EnsureAdvisoryIndexesAsync(cancellationToken), + EnsureDocumentsIndexesAsync(cancellationToken), + EnsureDtoIndexesAsync(cancellationToken), + EnsureAliasIndexesAsync(cancellationToken), + EnsureAffectedIndexesAsync(cancellationToken), + EnsureReferenceIndexesAsync(cancellationToken), + EnsureSourceStateIndexesAsync(cancellationToken), + EnsurePsirtFlagIndexesAsync(cancellationToken), + EnsureChangeHistoryIndexesAsync(cancellationToken), + EnsureGridFsIndexesAsync(cancellationToken)).ConfigureAwait(false); + + await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false); + + _logger.LogInformation("Mongo bootstrapper completed"); + } + + private async Task> ListCollectionsAsync(CancellationToken cancellationToken) + { + using var cursor = await _database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var list = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + return new HashSet(list, StringComparer.Ordinal); + } + + private Task EnsureLocksIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Locks); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("ttlAt"), + new CreateIndexOptions { Name = "ttl_at_ttl", ExpireAfter = TimeSpan.Zero }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureJobsIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Jobs); + var indexes = new List> + { + new( + Builders.IndexKeys.Descending("createdAt"), + new CreateIndexOptions { Name = "jobs_createdAt_desc" }), + new( + Builders.IndexKeys.Ascending("kind").Descending("createdAt"), + new CreateIndexOptions { Name = "jobs_kind_createdAt" }), + new( + Builders.IndexKeys.Ascending("status").Descending("createdAt"), + new CreateIndexOptions { Name = "jobs_status_createdAt" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureAdvisoryIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Advisory); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("advisoryKey"), + new CreateIndexOptions { Name = "advisory_key_unique", Unique = true }), + new( + Builders.IndexKeys.Descending("modified"), + new CreateIndexOptions { Name = "advisory_modified_desc" }), + new( + Builders.IndexKeys.Descending("published"), + new CreateIndexOptions { Name = "advisory_published_desc" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureDocumentsIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Document); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("sourceName").Ascending("uri"), + new CreateIndexOptions { Name = "document_source_uri_unique", Unique = true }), + new( + Builders.IndexKeys.Descending("fetchedAt"), + new CreateIndexOptions { Name = "document_fetchedAt_desc" }), + }; + + var expiresKey = Builders.IndexKeys.Ascending("expiresAt"); + var expiresOptions = new CreateIndexOptions + { + Name = _options.RawDocumentRetention > TimeSpan.Zero ? "document_expiresAt_ttl" : "document_expiresAt", + PartialFilterExpression = Builders.Filter.Exists("expiresAt", true), + }; + + if (_options.RawDocumentRetention > TimeSpan.Zero) + { + expiresOptions.ExpireAfter = TimeSpan.Zero; + } + + indexes.Add(new CreateIndexModel(expiresKey, expiresOptions)); + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureAliasIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Alias); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("scheme").Ascending("value"), + new CreateIndexOptions { Name = "alias_scheme_value", Unique = false }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureGridFsIndexesAsync(CancellationToken cancellationToken) + { + if (_options.RawDocumentRetention <= TimeSpan.Zero) + { + return Task.CompletedTask; + } + + var collectionName = $"{RawDocumentBucketName}.files"; + var collection = _database.GetCollection(collectionName); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("metadata.expiresAt"), + new CreateIndexOptions + { + Name = "gridfs_files_expiresAt_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = Builders.Filter.Exists("metadata.expiresAt", true), + }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureAffectedIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Affected); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("platform").Ascending("name"), + new CreateIndexOptions { Name = "affected_platform_name" }), + new( + Builders.IndexKeys.Ascending("advisoryId"), + new CreateIndexOptions { Name = "affected_advisoryId" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureReferenceIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Reference); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("url"), + new CreateIndexOptions { Name = "reference_url" }), + new( + Builders.IndexKeys.Ascending("advisoryId"), + new CreateIndexOptions { Name = "reference_advisoryId" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureSourceStateIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.SourceState); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("sourceName"), + new CreateIndexOptions { Name = "source_state_unique", Unique = true }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureDtoIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Dto); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("documentId"), + new CreateIndexOptions { Name = "dto_documentId" }), + new( + Builders.IndexKeys.Ascending("sourceName").Descending("validatedAt"), + new CreateIndexOptions { Name = "dto_source_validated" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsurePsirtFlagIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("advisoryKey"), + new CreateIndexOptions { Name = "psirt_advisoryKey_unique", Unique = true }), + new( + Builders.IndexKeys.Ascending("vendor"), + new CreateIndexOptions { Name = "psirt_vendor" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureChangeHistoryIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.ChangeHistory); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("source").Ascending("advisoryKey").Descending("capturedAt"), + new CreateIndexOptions { Name = "history_source_advisory_capturedAt" }), + new( + Builders.IndexKeys.Descending("capturedAt"), + new CreateIndexOptions { Name = "history_capturedAt" }), + new( + Builders.IndexKeys.Ascending("documentId"), + new CreateIndexOptions { Name = "history_documentId" }) + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs new file mode 100644 index 00000000..7b37cfb9 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed class MongoJobStore : IJobStore +{ + private static readonly string PendingStatus = JobRunStatus.Pending.ToString(); + private static readonly string RunningStatus = JobRunStatus.Running.ToString(); + + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public MongoJobStore(IMongoCollection collection, ILogger logger) + { + _collection = collection ?? throw new ArgumentNullException(nameof(collection)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken) + { + var runId = Guid.NewGuid(); + var document = JobRunDocumentExtensions.FromRequest(request, runId); + + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Created job run {RunId} for {Kind} with trigger {Trigger}", runId, request.Kind, request.Trigger); + + return document.ToSnapshot(); + } + + public async Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) + { + var filter = Builders.Filter.Eq(x => x.Id, runId) + & Builders.Filter.Eq(x => x.Status, PendingStatus); + + var update = Builders.Update + .Set(x => x.Status, RunningStatus) + .Set(x => x.StartedAt, startedAt.UtcDateTime); + + var result = await _collection.FindOneAndUpdateAsync( + filter, + update, + new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }, + cancellationToken).ConfigureAwait(false); + + if (result is null) + { + _logger.LogDebug("Failed to start job run {RunId}; status transition rejected", runId); + return null; + } + + return result.ToSnapshot(); + } + + public async Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) + { + var filter = Builders.Filter.Eq(x => x.Id, runId) + & Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); + + var update = Builders.Update + .Set(x => x.Status, completion.Status.ToString()) + .Set(x => x.CompletedAt, completion.CompletedAt.UtcDateTime) + .Set(x => x.Error, completion.Error); + + var result = await _collection.FindOneAndUpdateAsync( + filter, + update, + new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }, + cancellationToken).ConfigureAwait(false); + + if (result is null) + { + _logger.LogWarning("Failed to mark job run {RunId} as {Status}", runId, completion.Status); + return null; + } + + return result.ToSnapshot(); + } + + public async Task FindAsync(Guid runId, CancellationToken cancellationToken) + { + var cursor = await _collection.FindAsync(x => x.Id == runId, cancellationToken: cancellationToken).ConfigureAwait(false); + var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToSnapshot(); + } + + public async Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + { + if (limit <= 0) + { + return Array.Empty(); + } + + var filter = string.IsNullOrWhiteSpace(kind) + ? Builders.Filter.Empty + : Builders.Filter.Eq(x => x.Kind, kind); + + var cursor = await _collection.Find(filter) + .SortByDescending(x => x.CreatedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static doc => doc.ToSnapshot()).ToArray(); + } + + public async Task> GetActiveRunsAsync(CancellationToken cancellationToken) + { + var filter = Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); + var cursor = await _collection.Find(filter) + .SortByDescending(x => x.CreatedAt) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static doc => doc.ToSnapshot()).ToArray(); + } + + public async Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + { + var cursor = await _collection.Find(x => x.Kind == kind) + .SortByDescending(x => x.CreatedAt) + .Limit(1) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.FirstOrDefault()?.ToSnapshot(); + } + + public async Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + { + if (kinds is null) + { + throw new ArgumentNullException(nameof(kinds)); + } + + var kindList = kinds + .Where(static kind => !string.IsNullOrWhiteSpace(kind)) + .Select(static kind => kind.Trim()) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (kindList.Length == 0) + { + return new Dictionary(StringComparer.Ordinal); + } + + var matchStage = new BsonDocument("$match", new BsonDocument("kind", new BsonDocument("$in", new BsonArray(kindList)))); + var sortStage = new BsonDocument("$sort", new BsonDocument("createdAt", -1)); + var groupStage = new BsonDocument("$group", new BsonDocument + { + { "_id", "$kind" }, + { "document", new BsonDocument("$first", "$$ROOT") } + }); + + var pipeline = new[] { matchStage, sortStage, groupStage }; + + var aggregate = await _collection.Aggregate(pipeline) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var results = new Dictionary(StringComparer.Ordinal); + foreach (var element in aggregate) + { + if (!element.TryGetValue("_id", out var idValue) || idValue.BsonType != BsonType.String) + { + continue; + } + + if (!element.TryGetValue("document", out var documentValue) || documentValue.BsonType != BsonType.Document) + { + continue; + } + + var document = BsonSerializer.Deserialize(documentValue.AsBsonDocument); + results[idValue.AsString] = document.ToSnapshot(); + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs new file mode 100644 index 00000000..e2a3652b --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs @@ -0,0 +1,116 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed class MongoLeaseStore : ILeaseStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public MongoLeaseStore(IMongoCollection collection, ILogger logger) + { + _collection = collection ?? throw new ArgumentNullException(nameof(collection)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task TryAcquireAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + var nowUtc = now.UtcDateTime; + var ttlUtc = nowUtc.Add(leaseDuration); + + var filter = Builders.Filter.Eq(x => x.Key, key) + & Builders.Filter.Or( + Builders.Filter.Lte(x => x.TtlAt, nowUtc), + Builders.Filter.Eq(x => x.Holder, holder)); + + var update = Builders.Update + .Set(x => x.Holder, holder) + .Set(x => x.AcquiredAt, nowUtc) + .Set(x => x.HeartbeatAt, nowUtc) + .Set(x => x.LeaseMs, (long)leaseDuration.TotalMilliseconds) + .Set(x => x.TtlAt, ttlUtc); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }; + + var updated = await _collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false); + if (updated is not null) + { + _logger.LogDebug("Lease {Key} acquired by {Holder}", key, holder); + return updated.ToLease(); + } + + try + { + var document = new JobLeaseDocument + { + Key = key, + Holder = holder, + AcquiredAt = nowUtc, + HeartbeatAt = nowUtc, + LeaseMs = (long)leaseDuration.TotalMilliseconds, + TtlAt = ttlUtc, + }; + + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Lease {Key} inserted for {Holder}", key, holder); + return document.ToLease(); + } + catch (MongoWriteException ex) when (ex.WriteError.Category == ServerErrorCategory.DuplicateKey) + { + _logger.LogDebug(ex, "Lease {Key} already held by another process", key); + return null; + } + } + + public async Task HeartbeatAsync(string key, string holder, TimeSpan leaseDuration, DateTimeOffset now, CancellationToken cancellationToken) + { + var nowUtc = now.UtcDateTime; + var ttlUtc = nowUtc.Add(leaseDuration); + + var filter = Builders.Filter.Eq(x => x.Key, key) + & Builders.Filter.Eq(x => x.Holder, holder); + + var update = Builders.Update + .Set(x => x.HeartbeatAt, nowUtc) + .Set(x => x.LeaseMs, (long)leaseDuration.TotalMilliseconds) + .Set(x => x.TtlAt, ttlUtc); + + var updated = await _collection.FindOneAndUpdateAsync( + filter, + update, + new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }, + cancellationToken).ConfigureAwait(false); + + if (updated is null) + { + _logger.LogDebug("Heartbeat rejected for lease {Key} held by {Holder}", key, holder); + } + + return updated?.ToLease(); + } + + public async Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken) + { + var result = await _collection.DeleteOneAsync( + Builders.Filter.Eq(x => x.Key, key) + & Builders.Filter.Eq(x => x.Holder, holder), + cancellationToken).ConfigureAwait(false); + + if (result.DeletedCount == 0) + { + _logger.LogDebug("Lease {Key} not released by {Holder}; no matching document", key, holder); + return false; + } + + _logger.LogDebug("Lease {Key} released by {Holder}", key, holder); + return true; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs new file mode 100644 index 00000000..234aa370 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs @@ -0,0 +1,112 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed class MongoSourceStateRepository : ISourceStateRepository +{ + private readonly IMongoCollection _collection; + private const int MaxFailureReasonLength = 1024; + + private readonly ILogger _logger; + + public MongoSourceStateRepository(IMongoDatabase database, ILogger logger) + { + _collection = (database ?? throw new ArgumentNullException(nameof(database))) + .GetCollection(MongoStorageDefaults.Collections.SourceState); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task TryGetAsync(string sourceName, CancellationToken cancellationToken) + { + var cursor = await _collection.FindAsync(x => x.SourceName == sourceName, cancellationToken: cancellationToken).ConfigureAwait(false); + var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task UpsertAsync(SourceStateRecord record, CancellationToken cancellationToken) + { + var document = SourceStateDocumentExtensions.FromRecord(record with { UpdatedAt = DateTimeOffset.UtcNow }); + await _collection.ReplaceOneAsync( + x => x.SourceName == record.SourceName, + document, + new ReplaceOptions { IsUpsert = true }, + cancellationToken).ConfigureAwait(false); + + _logger.LogDebug("Upserted source state for {Source}", record.SourceName); + return document.ToRecord(); + } + + public async Task UpdateCursorAsync(string sourceName, BsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + var update = Builders.Update + .Set(x => x.Cursor, cursor ?? new BsonDocument()) + .Set(x => x.LastSuccess, completedAt.UtcDateTime) + .Set(x => x.FailCount, 0) + .Set(x => x.BackoffUntil, (DateTime?)null) + .Set(x => x.LastFailureReason, null) + .Set(x => x.UpdatedAt, DateTime.UtcNow) + .SetOnInsert(x => x.SourceName, sourceName); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + IsUpsert = true, + }; + + var document = await _collection + .FindOneAndUpdateAsync( + x => x.SourceName == sourceName, + update, + options, + cancellationToken) + .ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task MarkFailureAsync(string sourceName, DateTimeOffset failedAt, TimeSpan? backoff, string? failureReason, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + var reasonValue = NormalizeFailureReason(failureReason); + var update = Builders.Update + .Inc(x => x.FailCount, 1) + .Set(x => x.LastFailure, failedAt.UtcDateTime) + .Set(x => x.BackoffUntil, backoff.HasValue ? failedAt.UtcDateTime.Add(backoff.Value) : null) + .Set(x => x.LastFailureReason, reasonValue) + .Set(x => x.UpdatedAt, DateTime.UtcNow) + .SetOnInsert(x => x.SourceName, sourceName); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + IsUpsert = true, + }; + + var document = await _collection + .FindOneAndUpdateAsync( + x => x.SourceName == sourceName, + update, + options, + cancellationToken) + .ConfigureAwait(false); + return document?.ToRecord(); + } + + private static string? NormalizeFailureReason(string? reason) + { + if (string.IsNullOrWhiteSpace(reason)) + { + return null; + } + + var trimmed = reason.Trim(); + if (trimmed.Length <= MaxFailureReasonLength) + { + return trimmed; + } + + return trimmed[..MaxFailureReasonLength]; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs new file mode 100644 index 00000000..db8d636e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs @@ -0,0 +1,28 @@ +namespace StellaOps.Feedser.Storage.Mongo; + +public static class MongoStorageDefaults +{ + public const string DefaultDatabaseName = "feedser"; + + public static class Collections + { + public const string Source = "source"; + public const string SourceState = "source_state"; + public const string Document = "document"; + public const string Dto = "dto"; + public const string Advisory = "advisory"; + public const string Alias = "alias"; + public const string Affected = "affected"; + public const string Reference = "reference"; + public const string KevFlag = "kev_flag"; + public const string RuFlags = "ru_flags"; + public const string JpFlags = "jp_flags"; + public const string PsirtFlags = "psirt_flags"; + public const string MergeEvent = "merge_event"; + public const string ExportState = "export_state"; + public const string Locks = "locks"; + public const string Jobs = "jobs"; + public const string Migrations = "schema_migrations"; + public const string ChangeHistory = "source_change_history"; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs b/src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs new file mode 100644 index 00000000..a3896b7d --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs @@ -0,0 +1,78 @@ +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed class MongoStorageOptions +{ + public string ConnectionString { get; set; } = string.Empty; + + public string? DatabaseName { get; set; } + + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Retention period for raw documents (document + DTO + GridFS payloads). + /// Set to to disable automatic expiry. + /// + public TimeSpan RawDocumentRetention { get; set; } = TimeSpan.FromDays(45); + + /// + /// Additional grace period applied on top of before TTL purges old rows. + /// Allows the retention background service to delete GridFS blobs first. + /// + public TimeSpan RawDocumentRetentionTtlGrace { get; set; } = TimeSpan.FromDays(1); + + /// + /// Interval between retention sweeps. Only used when is greater than zero. + /// + public TimeSpan RawDocumentRetentionSweepInterval { get; set; } = TimeSpan.FromHours(6); + + public string GetDatabaseName() + { + if (!string.IsNullOrWhiteSpace(DatabaseName)) + { + return DatabaseName.Trim(); + } + + if (!string.IsNullOrWhiteSpace(ConnectionString)) + { + var url = MongoUrl.Create(ConnectionString); + if (!string.IsNullOrWhiteSpace(url.DatabaseName)) + { + return url.DatabaseName; + } + } + + return MongoStorageDefaults.DefaultDatabaseName; + } + + public void EnsureValid() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + throw new InvalidOperationException("Mongo connection string is not configured."); + } + + if (CommandTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Command timeout must be greater than zero."); + } + + if (RawDocumentRetention < TimeSpan.Zero) + { + throw new InvalidOperationException("Raw document retention cannot be negative."); + } + + if (RawDocumentRetentionTtlGrace < TimeSpan.Zero) + { + throw new InvalidOperationException("Raw document retention TTL grace cannot be negative."); + } + + if (RawDocumentRetention > TimeSpan.Zero && RawDocumentRetentionSweepInterval <= TimeSpan.Zero) + { + throw new InvalidOperationException("Raw document retention sweep interval must be positive when retention is enabled."); + } + + _ = GetDatabaseName(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..34f9836a --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Feedser.Storage.Mongo.Tests")] diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs new file mode 100644 index 00000000..17cdf4e9 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +public interface IPsirtFlagStore +{ + Task UpsertAsync(PsirtFlagRecord record, CancellationToken cancellationToken); + + Task FindAsync(string advisoryKey, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs new file mode 100644 index 00000000..d0e9ebc8 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs @@ -0,0 +1,52 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +[BsonIgnoreExtraElements] +public sealed class PsirtFlagDocument +{ + [BsonId] + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("vendor")] + public string Vendor { get; set; } = string.Empty; + + [BsonElement("sourceName")] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("advisoryIdText")] + public string AdvisoryIdText { get; set; } = string.Empty; + + [BsonElement("flaggedAt")] + public DateTime FlaggedAt { get; set; } +} + +internal static class PsirtFlagDocumentExtensions +{ + public static PsirtFlagDocument FromRecord(PsirtFlagRecord record) + { + ArgumentNullException.ThrowIfNull(record); + + return new PsirtFlagDocument + { + AdvisoryKey = string.IsNullOrWhiteSpace(record.AdvisoryKey) ? record.AdvisoryIdText : record.AdvisoryKey, + Vendor = record.Vendor, + SourceName = record.SourceName, + AdvisoryIdText = record.AdvisoryIdText, + FlaggedAt = record.FlaggedAt.UtcDateTime, + }; + } + + public static PsirtFlagRecord ToRecord(this PsirtFlagDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + return new PsirtFlagRecord( + document.AdvisoryKey, + document.Vendor, + document.SourceName, + document.AdvisoryIdText, + DateTime.SpecifyKind(document.FlaggedAt, DateTimeKind.Utc)); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs new file mode 100644 index 00000000..e5673f13 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +/// +/// Describes a PSIRT precedence flag for a canonical advisory. +/// +public sealed record PsirtFlagRecord( + string AdvisoryKey, + string Vendor, + string SourceName, + string AdvisoryIdText, + DateTimeOffset FlaggedAt) +{ + public PsirtFlagRecord WithFlaggedAt(DateTimeOffset flaggedAt) + => this with { FlaggedAt = flaggedAt.ToUniversalTime() }; +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs new file mode 100644 index 00000000..22c0928a --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs @@ -0,0 +1,50 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; + +public sealed class PsirtFlagStore : IPsirtFlagStore +{ + private readonly IMongoCollection _collection; + private readonly ILogger _logger; + + public PsirtFlagStore(IMongoDatabase database, ILogger logger) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(logger); + + _collection = database.GetCollection(MongoStorageDefaults.Collections.PsirtFlags); + _logger = logger; + } + + public async Task UpsertAsync(PsirtFlagRecord record, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(record); + ArgumentException.ThrowIfNullOrEmpty(record.AdvisoryKey); + + var document = PsirtFlagDocumentExtensions.FromRecord(record); + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, record.AdvisoryKey); + var options = new ReplaceOptions { IsUpsert = true }; + + try + { + await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted PSIRT flag for {AdvisoryKey}", record.AdvisoryKey); + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + _logger.LogWarning(ex, "Duplicate PSIRT flag detected for {AdvisoryKey}", record.AdvisoryKey); + } + } + + public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs b/src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs new file mode 100644 index 00000000..2e5af614 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs @@ -0,0 +1,155 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; + +namespace StellaOps.Feedser.Storage.Mongo; + +/// +/// Periodically purges expired raw documents, associated DTO payloads, and GridFS content. +/// Complements TTL indexes by ensuring deterministic cleanup before Mongo's background sweeper runs. +/// +internal sealed class RawDocumentRetentionService : BackgroundService +{ + private readonly IMongoCollection _documents; + private readonly IMongoCollection _dtos; + private readonly GridFSBucket _bucket; + private readonly MongoStorageOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public RawDocumentRetentionService( + IMongoDatabase database, + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _documents = database.GetCollection(MongoStorageDefaults.Collections.Document); + _dtos = database.GetCollection(MongoStorageDefaults.Collections.Dto); + _bucket = new GridFSBucket(database, new GridFSBucketOptions + { + BucketName = "documents", + ReadConcern = database.Settings.ReadConcern, + WriteConcern = database.Settings.WriteConcern, + }); + + _options = options.Value; + _logger = logger; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (_options.RawDocumentRetention <= TimeSpan.Zero) + { + _logger.LogInformation("Raw document retention disabled; purge service idle."); + return; + } + + var sweepInterval = _options.RawDocumentRetentionSweepInterval > TimeSpan.Zero + ? _options.RawDocumentRetentionSweepInterval + : TimeSpan.FromHours(6); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await SweepExpiredDocumentsAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Raw document retention sweep failed"); + } + + try + { + await Task.Delay(sweepInterval, stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + } + + internal async Task SweepExpiredDocumentsAsync(CancellationToken cancellationToken) + { + var grace = _options.RawDocumentRetentionTtlGrace >= TimeSpan.Zero + ? _options.RawDocumentRetentionTtlGrace + : TimeSpan.Zero; + var threshold = _timeProvider.GetUtcNow() + grace; + + var filterBuilder = Builders.Filter; + var filter = filterBuilder.And( + filterBuilder.Ne(doc => doc.ExpiresAt, null), + filterBuilder.Lte(doc => doc.ExpiresAt, threshold.UtcDateTime)); + + var removed = 0; + + while (!cancellationToken.IsCancellationRequested) + { + var batch = await _documents + .Find(filter) + .SortBy(doc => doc.ExpiresAt) + .Limit(200) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + if (batch.Count == 0) + { + break; + } + + foreach (var document in batch) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + await PurgeDocumentAsync(document, cancellationToken).ConfigureAwait(false); + removed++; + } + } + + if (removed > 0) + { + _logger.LogInformation("Purged {Count} expired raw documents (threshold <= {Threshold})", removed, threshold); + } + + return removed; + } + + private async Task PurgeDocumentAsync(DocumentDocument document, CancellationToken cancellationToken) + { + if (document.GridFsId.HasValue) + { + try + { + await _bucket.DeleteAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (GridFSFileNotFoundException) + { + // already removed or TTL swept + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to delete GridFS payload {GridFsId} for document {DocumentId}", document.GridFsId, document.Id); + } + } + + await _dtos.DeleteManyAsync(x => x.DocumentId == document.Id, cancellationToken).ConfigureAwait(false); + await _documents.DeleteOneAsync(x => x.Id == document.Id, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..3ec0f89e --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs @@ -0,0 +1,88 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Feedser.Storage.Mongo.ChangeHistory; +using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Feedser.Storage.Mongo.JpFlags; +using StellaOps.Feedser.Storage.Mongo.MergeEvents; +using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Feedser.Storage.Mongo.Migrations; + +namespace StellaOps.Feedser.Storage.Mongo; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddMongoStorage(this IServiceCollection services, Action configureOptions) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configureOptions); + + services.AddOptions() + .Configure(configureOptions) + .PostConfigure(static options => options.EnsureValid()); + + services.TryAddSingleton(TimeProvider.System); + + services.AddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + return new MongoClient(options.ConnectionString); + }); + + services.AddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var client = sp.GetRequiredService(); + var settings = new MongoDatabaseSettings + { + ReadConcern = ReadConcern.Majority, + WriteConcern = WriteConcern.WMajority, + ReadPreference = ReadPreference.PrimaryPreferred, + }; + + var database = client.GetDatabase(options.GetDatabaseName(), settings); + var writeConcern = database.Settings.WriteConcern.With(wTimeout: options.CommandTimeout); + return database.WithWriteConcern(writeConcern); + }); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.TryAddSingleton(); + + services.AddSingleton>(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(MongoStorageDefaults.Collections.Jobs); + }); + + services.AddSingleton>(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(MongoStorageDefaults.Collections.Locks); + }); + + services.AddHostedService(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs b/src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs new file mode 100644 index 00000000..489951d0 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs @@ -0,0 +1,73 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Feedser.Storage.Mongo; + +[BsonIgnoreExtraElements] +public sealed class SourceStateDocument +{ + [BsonId] + public string SourceName { get; set; } = string.Empty; + + [BsonElement("enabled")] + public bool Enabled { get; set; } = true; + + [BsonElement("paused")] + public bool Paused { get; set; } + + [BsonElement("cursor")] + public BsonDocument Cursor { get; set; } = new(); + + [BsonElement("lastSuccess")] + [BsonIgnoreIfNull] + public DateTime? LastSuccess { get; set; } + + [BsonElement("lastFailure")] + [BsonIgnoreIfNull] + public DateTime? LastFailure { get; set; } + + [BsonElement("failCount")] + public int FailCount { get; set; } + + [BsonElement("backoffUntil")] + [BsonIgnoreIfNull] + public DateTime? BackoffUntil { get; set; } + + [BsonElement("updatedAt")] + public DateTime UpdatedAt { get; set; } + + [BsonElement("lastFailureReason")] + [BsonIgnoreIfNull] + public string? LastFailureReason { get; set; } +} + +internal static class SourceStateDocumentExtensions +{ + public static SourceStateDocument FromRecord(SourceStateRecord record) + => new() + { + SourceName = record.SourceName, + Enabled = record.Enabled, + Paused = record.Paused, + Cursor = record.Cursor ?? new BsonDocument(), + LastSuccess = record.LastSuccess?.UtcDateTime, + LastFailure = record.LastFailure?.UtcDateTime, + FailCount = record.FailCount, + BackoffUntil = record.BackoffUntil?.UtcDateTime, + UpdatedAt = record.UpdatedAt.UtcDateTime, + LastFailureReason = record.LastFailureReason, + }; + + public static SourceStateRecord ToRecord(this SourceStateDocument document) + => new( + document.SourceName, + document.Enabled, + document.Paused, + document.Cursor ?? new BsonDocument(), + document.LastSuccess.HasValue ? DateTime.SpecifyKind(document.LastSuccess.Value, DateTimeKind.Utc) : null, + document.LastFailure.HasValue ? DateTime.SpecifyKind(document.LastFailure.Value, DateTimeKind.Utc) : null, + document.FailCount, + document.BackoffUntil.HasValue ? DateTime.SpecifyKind(document.BackoffUntil.Value, DateTimeKind.Utc) : null, + DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc), + document.LastFailureReason); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs b/src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs new file mode 100644 index 00000000..f86cd0d6 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs @@ -0,0 +1,15 @@ +using MongoDB.Bson; + +namespace StellaOps.Feedser.Storage.Mongo; + +public sealed record SourceStateRecord( + string SourceName, + bool Enabled, + bool Paused, + BsonDocument Cursor, + DateTimeOffset? LastSuccess, + DateTimeOffset? LastFailure, + int FailCount, + DateTimeOffset? BackoffUntil, + DateTimeOffset UpdatedAt, + string? LastFailureReason); diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs b/src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs new file mode 100644 index 00000000..bfa165e8 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs @@ -0,0 +1,19 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Feedser.Storage.Mongo; + +public static class SourceStateRepositoryExtensions +{ + public static Task MarkFailureAsync( + this ISourceStateRepository repository, + string sourceName, + DateTimeOffset failedAt, + TimeSpan? backoff, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(repository); + return repository.MarkFailureAsync(sourceName, failedAt, backoff, failureReason: null, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj b/src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj new file mode 100644 index 00000000..bc8b30e3 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Storage.Mongo/TASKS.md b/src/StellaOps.Feedser.Storage.Mongo/TASKS.md new file mode 100644 index 00000000..f40a1745 --- /dev/null +++ b/src/StellaOps.Feedser.Storage.Mongo/TASKS.md @@ -0,0 +1,15 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|MongoBootstrapper to create collections/indexes|BE-Storage|Storage.Mongo|DONE – `MongoBootstrapper` ensures collections & indexes incl. TTL on locks.ttlAt.| +|SourceState repository (get/set/backoff)|BE-Conn-Base|Storage.Mongo|DONE – implemented `MongoSourceStateRepository`.| +|Document/DTO stores with SHA/metadata|BE-Conn-Base|Storage.Mongo|DONE – DocumentStore and DtoStore provide upsert/status lookups.| +|AdvisoryStore (GetAllAsync etc.)|BE-Export|Models|DONE – AdvisoryStore handles upsert + recent/advisory fetches.| +|Job store (runs/active/recent)|BE-Core|Storage.Mongo|DONE – `MongoJobStore` covers create/start/complete queries.| +|Alias and reference secondary indexes|BE-Storage|Models|DONE – bootstrapper builds alias/reference indexes.| +|MergeEvent store|BE-Merge|Models|DONE – MergeEventStore appends/retrieves recent events.| +|ExportState store|BE-Export|Exporters|DONE – ExportStateStore upserts and retrieves exporter metadata.| +|Performance tests for large advisories|QA|Storage.Mongo|DONE – `AdvisoryStorePerformanceTests` exercises large payload upsert/find throughput budgets.| +|Migration playbook for schema/index changes|BE-Storage|Storage.Mongo|DONE – `MongoMigrationRunner` executes `IMongoMigration` steps recorded in `schema_migrations`; see `MIGRATIONS.md`.| +|Raw document retention/TTL strategy|BE-Storage|Storage.Mongo|DONE – retention options flow into `RawDocumentRetentionService` and TTL migrations for `document`/GridFS indexes.| +|Persist last failure reason in SourceState|BE-Storage|Storage.Mongo|DONE – `MongoSourceStateRepository.MarkFailureAsync` stores `lastFailureReason` with length guard + reset on success.| diff --git a/src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs b/src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs new file mode 100644 index 00000000..d8cb8e12 --- /dev/null +++ b/src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs @@ -0,0 +1,118 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Feedser.Source.Common.Http; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Feedser.Source.Common; +using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.Testing; + +namespace StellaOps.Feedser.Testing; + +/// +/// Provides a reusable container for connector integration tests with canned HTTP responses and Mongo isolation. +/// +public sealed class ConnectorTestHarness : IAsyncDisposable +{ + private readonly MongoIntegrationFixture _fixture; + private readonly DateTimeOffset _initialTime; + private readonly string[] _httpClientNames; + private ServiceProvider? _serviceProvider; + + public ConnectorTestHarness(MongoIntegrationFixture fixture, DateTimeOffset initialTime, params string[] httpClientNames) + { + _fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); + _initialTime = initialTime; + _httpClientNames = httpClientNames.Length == 0 + ? Array.Empty() + : httpClientNames.Distinct(StringComparer.Ordinal).ToArray(); + + TimeProvider = new FakeTimeProvider(initialTime) + { + AutoAdvanceAmount = TimeSpan.Zero, + }; + Handler = new CannedHttpMessageHandler(); + } + + public FakeTimeProvider TimeProvider { get; } + + public CannedHttpMessageHandler Handler { get; } + + public ServiceProvider ServiceProvider => _serviceProvider ?? throw new InvalidOperationException("Call EnsureServiceProviderAsync first."); + + public async Task EnsureServiceProviderAsync(Action configureServices) + { + ArgumentNullException.ThrowIfNull(configureServices); + + if (_serviceProvider is not null) + { + return _serviceProvider; + } + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(TimeProvider); + services.AddSingleton(Handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + + configureServices(services); + + foreach (var clientName in _httpClientNames) + { + services.Configure(clientName, options => + { + options.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = Handler; + }); + }); + } + + var provider = services.BuildServiceProvider(); + _serviceProvider = provider; + + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + public async Task ResetAsync() + { + if (_serviceProvider is { } provider) + { + if (provider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + provider.Dispose(); + } + + _serviceProvider = null; + } + + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + Handler.Clear(); + TimeProvider.SetUtcNow(_initialTime); + } + + public async ValueTask DisposeAsync() + { + await ResetAsync(); + } +} diff --git a/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs b/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs new file mode 100644 index 00000000..d97e42ed --- /dev/null +++ b/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs @@ -0,0 +1,31 @@ +using MongoDB.Bson; +using Mongo2Go; +using Xunit; +using MongoDB.Driver; + +namespace StellaOps.Feedser.Testing; + +public sealed class MongoIntegrationFixture : IAsyncLifetime +{ + public MongoDbRunner Runner { get; private set; } = null!; + public IMongoDatabase Database { get; private set; } = null!; + public IMongoClient Client { get; private set; } = null!; + + public Task InitializeAsync() + { + Runner = MongoDbRunner.Start(singleNodeReplSet: true); + var settings = MongoClientSettings.FromConnectionString(Runner.ConnectionString); +#pragma warning disable CS0618 + settings.GuidRepresentation = GuidRepresentation.Standard; +#pragma warning restore CS0618 + Client = new MongoClient(settings); + Database = Client.GetDatabase($"feedser-tests-{Guid.NewGuid():N}"); + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + Runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj b/src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj new file mode 100644 index 00000000..1a3fdb0d --- /dev/null +++ b/src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + all + + + + + + + diff --git a/src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs b/src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs new file mode 100644 index 00000000..21712008 --- /dev/null +++ b/src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs b/src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs new file mode 100644 index 00000000..afcf2d3d --- /dev/null +++ b/src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs @@ -0,0 +1,6 @@ +using Xunit; + +namespace StellaOps.Feedser.Testing; + +[CollectionDefinition("mongo-fixture", DisableParallelization = true)] +public sealed class MongoFixtureCollection : ICollectionFixture; diff --git a/src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs b/src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs new file mode 100644 index 00000000..967888d4 --- /dev/null +++ b/src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs @@ -0,0 +1,29 @@ +using StellaOps.Plugin; + +namespace StellaOps.Feedser.WebService.Tests; + +public class PluginLoaderTests +{ + private sealed class NullServices : IServiceProvider + { + public object? GetService(Type serviceType) => null; + } + + [Fact] + public void ScansConnectorPluginsDirectory() + { + var services = new NullServices(); + var catalog = new PluginCatalog().AddFromDirectory(Path.Combine(AppContext.BaseDirectory, "PluginBinaries")); + var plugins = catalog.GetAvailableConnectorPlugins(services); + Assert.NotNull(plugins); + } + + [Fact] + public void ScansExporterPluginsDirectory() + { + var services = new NullServices(); + var catalog = new PluginCatalog().AddFromDirectory(Path.Combine(AppContext.BaseDirectory, "PluginBinaries")); + var plugins = catalog.GetAvailableExporterPlugins(services); + Assert.NotNull(plugins); + } +} diff --git a/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj b/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj new file mode 100644 index 00000000..739ea8e7 --- /dev/null +++ b/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs b/src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs new file mode 100644 index 00000000..7d6fe2ab --- /dev/null +++ b/src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs @@ -0,0 +1,346 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Mongo2Go; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.WebService.Jobs; +using StellaOps.Feedser.WebService.Options; +using Xunit.Sdk; + +namespace StellaOps.Feedser.WebService.Tests; + +public sealed class WebServiceEndpointsTests : IAsyncLifetime +{ + private MongoDbRunner _runner = null!; + private FeedserApplicationFactory _factory = null!; + + public Task InitializeAsync() + { + _runner = MongoDbRunner.Start(singleNodeReplSet: true); + _factory = new FeedserApplicationFactory(_runner.ConnectionString); + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + _factory.Dispose(); + _runner.Dispose(); + return Task.CompletedTask; + } + + [Fact] + public async Task HealthAndReadyEndpointsRespond() + { + using var client = _factory.CreateClient(); + + var healthResponse = await client.GetAsync("/health"); + if (!healthResponse.IsSuccessStatusCode) + { + var body = await healthResponse.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"/health failed: {(int)healthResponse.StatusCode} {body}"); + } + + var readyResponse = await client.GetAsync("/ready"); + if (!readyResponse.IsSuccessStatusCode) + { + var body = await readyResponse.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"/ready failed: {(int)readyResponse.StatusCode} {body}"); + } + + var healthPayload = await healthResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(healthPayload); + Assert.Equal("healthy", healthPayload!.Status); + Assert.Equal("mongo", healthPayload.Storage.Driver); + + var readyPayload = await readyResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(readyPayload); + Assert.Equal("ready", readyPayload!.Status); + Assert.Equal("ready", readyPayload.Mongo.Status); + } + + [Fact] + public async Task JobsEndpointsReturnExpectedStatuses() + { + using var client = _factory.CreateClient(); + + var definitions = await client.GetAsync("/jobs/definitions"); + if (!definitions.IsSuccessStatusCode) + { + var body = await definitions.Content.ReadAsStringAsync(); + throw new Xunit.Sdk.XunitException($"/jobs/definitions failed: {(int)definitions.StatusCode} {body}"); + } + + var trigger = await client.PostAsync("/jobs/unknown", new StringContent("{}", System.Text.Encoding.UTF8, "application/json")); + Assert.Equal(HttpStatusCode.NotFound, trigger.StatusCode); + var problem = await trigger.Content.ReadFromJsonAsync(); + Assert.NotNull(problem); + Assert.Equal("https://stellaops.org/problems/not-found", problem!.Type); + Assert.Equal(404, problem.Status); + } + + [Fact] + public async Task JobRunEndpointReturnsProblemWhenNotFound() + { + using var client = _factory.CreateClient(); + var response = await client.GetAsync($"/jobs/{Guid.NewGuid()}"); + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + var problem = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(problem); + Assert.Equal("https://stellaops.org/problems/not-found", problem!.Type); + } + + [Fact] + public async Task JobTriggerMapsCoordinatorOutcomes() + { + var handler = _factory.Services.GetRequiredService(); + using var client = _factory.CreateClient(); + + handler.NextResult = JobTriggerResult.AlreadyRunning("busy"); + var conflict = await client.PostAsync("/jobs/test", JsonContent.Create(new JobTriggerRequest())); + Assert.Equal(HttpStatusCode.Conflict, conflict.StatusCode); + var conflictProblem = await conflict.Content.ReadFromJsonAsync(); + Assert.NotNull(conflictProblem); + Assert.Equal("https://stellaops.org/problems/conflict", conflictProblem!.Type); + + handler.NextResult = JobTriggerResult.Accepted(new JobRunSnapshot(Guid.NewGuid(), "demo", JobRunStatus.Pending, DateTimeOffset.UtcNow, null, null, "api", null, null, null, null, new Dictionary())); + var accepted = await client.PostAsync("/jobs/test", JsonContent.Create(new JobTriggerRequest())); + Assert.Equal(HttpStatusCode.Accepted, accepted.StatusCode); + Assert.NotNull(accepted.Headers.Location); + var acceptedPayload = await accepted.Content.ReadFromJsonAsync(); + Assert.NotNull(acceptedPayload); + + handler.NextResult = JobTriggerResult.Failed(new JobRunSnapshot(Guid.NewGuid(), "demo", JobRunStatus.Failed, DateTimeOffset.UtcNow, null, DateTimeOffset.UtcNow, "api", null, "err", null, null, new Dictionary()), "boom"); + var failed = await client.PostAsync("/jobs/test", JsonContent.Create(new JobTriggerRequest())); + Assert.Equal(HttpStatusCode.InternalServerError, failed.StatusCode); + var failureProblem = await failed.Content.ReadFromJsonAsync(); + Assert.NotNull(failureProblem); + Assert.Equal("https://stellaops.org/problems/job-failure", failureProblem!.Type); + } + + [Fact] + public async Task JobsEndpointsExposeJobData() + { + var handler = _factory.Services.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + var run = new JobRunSnapshot( + Guid.NewGuid(), + "demo", + JobRunStatus.Succeeded, + now, + now, + now.AddSeconds(2), + "api", + "hash", + null, + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(1), + new Dictionary { ["key"] = "value" }); + + handler.Definitions = new[] + { + new JobDefinition("demo", typeof(DemoJob), TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(1), "*/5 * * * *", true) + }; + handler.LastRuns["demo"] = run; + handler.RecentRuns = new[] { run }; + handler.ActiveRuns = Array.Empty(); + handler.Runs[run.RunId] = run; + + try + { + using var client = _factory.CreateClient(); + + var definitions = await client.GetFromJsonAsync>("/jobs/definitions"); + Assert.NotNull(definitions); + Assert.Single(definitions!); + Assert.Equal("demo", definitions![0].Kind); + Assert.NotNull(definitions[0].LastRun); + Assert.Equal(run.RunId, definitions[0].LastRun!.RunId); + + var runPayload = await client.GetFromJsonAsync($"/jobs/{run.RunId}"); + Assert.NotNull(runPayload); + Assert.Equal(run.RunId, runPayload!.RunId); + Assert.Equal("Succeeded", runPayload.Status); + + var runs = await client.GetFromJsonAsync>("/jobs?kind=demo&limit=5"); + Assert.NotNull(runs); + Assert.Single(runs!); + Assert.Equal(run.RunId, runs![0].RunId); + + var runsByDefinition = await client.GetFromJsonAsync>("/jobs/definitions/demo/runs"); + Assert.NotNull(runsByDefinition); + Assert.Single(runsByDefinition!); + + var active = await client.GetFromJsonAsync>("/jobs/active"); + Assert.NotNull(active); + Assert.Empty(active!); + } + finally + { + handler.Definitions = Array.Empty(); + handler.RecentRuns = Array.Empty(); + handler.ActiveRuns = Array.Empty(); + handler.Runs.Clear(); + handler.LastRuns.Clear(); + } + } + + private sealed class FeedserApplicationFactory : WebApplicationFactory + { + private readonly string _connectionString; + private readonly string? _previousDsn; + private readonly string? _previousDriver; + private readonly string? _previousTimeout; + private readonly string? _previousTelemetryEnabled; + private readonly string? _previousTelemetryLogging; + private readonly string? _previousTelemetryTracing; + private readonly string? _previousTelemetryMetrics; + + public FeedserApplicationFactory(string connectionString) + { + _connectionString = connectionString; + _previousDsn = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__DSN"); + _previousDriver = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__DRIVER"); + _previousTimeout = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS"); + _previousTelemetryEnabled = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED"); + _previousTelemetryLogging = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING"); + _previousTelemetryTracing = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING"); + _previousTelemetryMetrics = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS"); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DSN", connectionString); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DRIVER", "mongo"); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS", "30"); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED", "false"); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING", "false"); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING", "false"); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS", "false"); + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.ConfigureAppConfiguration((context, configurationBuilder) => + { + var settings = new Dictionary + { + ["Plugins:Directory"] = Path.Combine(context.HostingEnvironment.ContentRootPath, "PluginBinaries"), + }; + + configurationBuilder.AddInMemoryCollection(settings!); + }); + + builder.ConfigureServices(services => + { + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.PostConfigure(options => + { + options.Storage.Driver = "mongo"; + options.Storage.Dsn = _connectionString; + options.Storage.CommandTimeoutSeconds = 30; + options.Plugins.Directory ??= Path.Combine(AppContext.BaseDirectory, "PluginBinaries"); + options.Telemetry.Enabled = false; + options.Telemetry.EnableLogging = false; + options.Telemetry.EnableTracing = false; + options.Telemetry.EnableMetrics = false; + }); + }); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DSN", _previousDsn); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DRIVER", _previousDriver); + Environment.SetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS", _previousTimeout); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED", _previousTelemetryEnabled); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING", _previousTelemetryLogging); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING", _previousTelemetryTracing); + Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS", _previousTelemetryMetrics); + } + } + + private sealed record HealthPayload(string Status, DateTimeOffset StartedAt, double UptimeSeconds, StoragePayload Storage, TelemetryPayload Telemetry); + + private sealed record StoragePayload(string Driver, bool Completed, DateTimeOffset? CompletedAt, double? DurationMs); + + private sealed record TelemetryPayload(bool Enabled, bool Tracing, bool Metrics, bool Logging); + + private sealed record ReadyPayload(string Status, DateTimeOffset StartedAt, double UptimeSeconds, ReadyMongoPayload Mongo); + + private sealed record ReadyMongoPayload(string Status, double? LatencyMs, DateTimeOffset? CheckedAt, string? Error); + + private sealed record JobDefinitionPayload(string Kind, bool Enabled, string? CronExpression, TimeSpan Timeout, TimeSpan LeaseDuration, JobRunPayload? LastRun); + + private sealed record JobRunPayload(Guid RunId, string Kind, string Status, string Trigger, DateTimeOffset CreatedAt, DateTimeOffset? StartedAt, DateTimeOffset? CompletedAt, string? Error, TimeSpan? Duration, Dictionary Parameters); + + private sealed record ProblemDocument(string? Type, string? Title, int? Status, string? Detail, string? Instance); + + private sealed class DemoJob : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) => Task.CompletedTask; + } + + private sealed class StubJobCoordinator : IJobCoordinator + { + public JobTriggerResult NextResult { get; set; } = JobTriggerResult.NotFound("not set"); + + public IReadOnlyList Definitions { get; set; } = Array.Empty(); + + public IReadOnlyList RecentRuns { get; set; } = Array.Empty(); + + public IReadOnlyList ActiveRuns { get; set; } = Array.Empty(); + + public Dictionary Runs { get; } = new(); + + public Dictionary LastRuns { get; } = new(StringComparer.Ordinal); + + public Task TriggerAsync(string kind, IReadOnlyDictionary? parameters, string trigger, CancellationToken cancellationToken) + => Task.FromResult(NextResult); + + public Task> GetDefinitionsAsync(CancellationToken cancellationToken) + => Task.FromResult(Definitions); + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + { + IEnumerable query = RecentRuns; + if (!string.IsNullOrWhiteSpace(kind)) + { + query = query.Where(run => string.Equals(run.Kind, kind, StringComparison.Ordinal)); + } + + return Task.FromResult>(query.Take(limit).ToArray()); + } + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + => Task.FromResult(ActiveRuns); + + public Task GetRunAsync(Guid runId, CancellationToken cancellationToken) + => Task.FromResult(Runs.TryGetValue(runId, out var run) ? run : null); + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + => Task.FromResult(LastRuns.TryGetValue(kind, out var run) ? run : null); + + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var kind in kinds) + { + if (kind is null) + { + continue; + } + + if (LastRuns.TryGetValue(kind, out var run) && run is not null) + { + map[kind] = run; + } + } + + return Task.FromResult>(map); + } + } +} diff --git a/src/StellaOps.Feedser.WebService/AGENTS.md b/src/StellaOps.Feedser.WebService/AGENTS.md new file mode 100644 index 00000000..20ba4c4e --- /dev/null +++ b/src/StellaOps.Feedser.WebService/AGENTS.md @@ -0,0 +1,36 @@ +# AGENTS +## Role +Minimal API host wiring configuration, storage, plugin routines, and job endpoints. Operational surface for health, readiness, and job control. +## Scope +- Configuration: appsettings.json + etc/feedser.yaml (yaml path = ../etc/feedser.yaml); bind into FeedserOptions with validation (Only Mongo supported). +- Mongo: MongoUrl from options.Storage.Dsn; IMongoClient/IMongoDatabase singletons; default database name fallback (options -> URL -> "feedser"). +- Services: AddMongoStorage(); AddSourceHttpClients(); RegisterPluginRoutines(configuration, PluginHostOptions). +- Bootstrap: MongoBootstrapper.InitializeAsync on startup. +- Endpoints: + - GET / -> "Hello World!" (placeholder). + - GET /health -> {status:"healthy"} after options validation binds. + - GET /ready -> MongoDB ping; 503 on MongoException/Timeout. + - GET /jobs?kind=&limit= -> recent runs. + - GET /jobs/{id} -> run detail. + - GET /jobs/definitions -> definitions with lastRun. + - GET /jobs/definitions/{kind} -> definition + lastRun or 404. + - GET /jobs/definitions/{kind}/runs?limit= -> recent runs or 404 if kind unknown. + - GET /jobs/active -> currently running. + - POST /jobs/{*jobKind} with {trigger?,parameters?} -> 202 Accepted (Location:/jobs/{runId}) | 404 | 409 | 423. +- PluginHost defaults: BaseDirectory = solution root; PluginsDirectory = "PluginBinaries"; SearchPatterns += "StellaOps.Feedser.Plugin.*.dll"; EnsureDirectoryExists = true. +## Participants +- Core job system; Storage.Mongo; Source.Common HTTP clients; Exporter and Connector plugin routines discover/register jobs. +## Interfaces & contracts +- Dependency injection boundary for all connectors/exporters; IOptions validated on start. +- Cancellation: pass app.Lifetime.ApplicationStopping to bootstrapper. +## In/Out of scope +In: hosting, DI composition, REST surface, readiness checks. +Out: business logic of jobs, HTML UI, authn/z (future). +## Observability & security expectations +- Log startup config (redact DSN credentials), plugin scan results (missing ordered plugins if any). +- Structured responses with status codes; no stack traces in HTTP bodies; errors mapped cleanly. +## Tests +- Author and review coverage in `../StellaOps.Feedser.WebService.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs b/src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs new file mode 100644 index 00000000..1c78f040 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs @@ -0,0 +1,32 @@ +namespace StellaOps.Feedser.WebService.Diagnostics; + +internal sealed record StorageBootstrapHealth( + string Driver, + bool Completed, + DateTimeOffset? CompletedAt, + double? DurationMs); + +internal sealed record TelemetryHealth( + bool Enabled, + bool Tracing, + bool Metrics, + bool Logging); + +internal sealed record HealthDocument( + string Status, + DateTimeOffset StartedAt, + double UptimeSeconds, + StorageBootstrapHealth Storage, + TelemetryHealth Telemetry); + +internal sealed record MongoReadyHealth( + string Status, + double? LatencyMs, + DateTimeOffset? CheckedAt, + string? Error); + +internal sealed record ReadyDocument( + string Status, + DateTimeOffset StartedAt, + double UptimeSeconds, + MongoReadyHealth Mongo); diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs b/src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs new file mode 100644 index 00000000..b53a241c --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs @@ -0,0 +1,25 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Feedser.WebService.Diagnostics; + +internal static class JobMetrics +{ + internal const string MeterName = "StellaOps.Feedser.WebService.Jobs"; + + private static readonly Meter Meter = new(MeterName); + + internal static readonly Counter TriggerCounter = Meter.CreateCounter( + "web.jobs.triggered", + unit: "count", + description: "Number of job trigger requests accepted by the web service."); + + internal static readonly Counter TriggerConflictCounter = Meter.CreateCounter( + "web.jobs.trigger.conflict", + unit: "count", + description: "Number of job trigger requests that resulted in conflicts or rejections."); + + internal static readonly Counter TriggerFailureCounter = Meter.CreateCounter( + "web.jobs.trigger.failed", + unit: "count", + description: "Number of job trigger requests that failed at runtime."); +} diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs b/src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs new file mode 100644 index 00000000..5d921c4a --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Feedser.WebService.Diagnostics; + +internal static class ProblemTypes +{ + public const string NotFound = "https://stellaops.org/problems/not-found"; + public const string Validation = "https://stellaops.org/problems/validation"; + public const string Conflict = "https://stellaops.org/problems/conflict"; + public const string Locked = "https://stellaops.org/problems/locked"; + public const string LeaseRejected = "https://stellaops.org/problems/lease-rejected"; + public const string JobFailure = "https://stellaops.org/problems/job-failure"; + public const string ServiceUnavailable = "https://stellaops.org/problems/service-unavailable"; +} diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs b/src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs new file mode 100644 index 00000000..017f25b3 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs @@ -0,0 +1,74 @@ +using System.Diagnostics; + +namespace StellaOps.Feedser.WebService.Diagnostics; + +internal sealed class ServiceStatus +{ + private readonly TimeProvider _timeProvider; + private readonly DateTimeOffset _startedAt; + private readonly object _sync = new(); + + private DateTimeOffset? _bootstrapCompletedAt; + private TimeSpan? _bootstrapDuration; + private DateTimeOffset? _lastReadyCheckAt; + private TimeSpan? _lastMongoLatency; + private string? _lastMongoError; + private bool _lastReadySucceeded; + + public ServiceStatus(TimeProvider timeProvider) + { + _timeProvider = timeProvider ?? TimeProvider.System; + _startedAt = _timeProvider.GetUtcNow(); + } + + public ServiceHealthSnapshot CreateSnapshot() + { + lock (_sync) + { + return new ServiceHealthSnapshot( + CapturedAt: _timeProvider.GetUtcNow(), + StartedAt: _startedAt, + BootstrapCompletedAt: _bootstrapCompletedAt, + BootstrapDuration: _bootstrapDuration, + LastReadyCheckAt: _lastReadyCheckAt, + LastMongoLatency: _lastMongoLatency, + LastMongoError: _lastMongoError, + LastReadySucceeded: _lastReadySucceeded); + } + } + + public void MarkBootstrapCompleted(TimeSpan duration) + { + lock (_sync) + { + var completedAt = _timeProvider.GetUtcNow(); + _bootstrapCompletedAt = completedAt; + _bootstrapDuration = duration; + _lastReadySucceeded = true; + _lastMongoLatency = duration; + _lastMongoError = null; + _lastReadyCheckAt = completedAt; + } + } + + public void RecordMongoCheck(bool success, TimeSpan latency, string? error) + { + lock (_sync) + { + _lastReadySucceeded = success; + _lastMongoLatency = latency; + _lastMongoError = success ? null : error; + _lastReadyCheckAt = _timeProvider.GetUtcNow(); + } + } +} + +internal sealed record ServiceHealthSnapshot( + DateTimeOffset CapturedAt, + DateTimeOffset StartedAt, + DateTimeOffset? BootstrapCompletedAt, + TimeSpan? BootstrapDuration, + DateTimeOffset? LastReadyCheckAt, + TimeSpan? LastMongoLatency, + string? LastMongoError, + bool LastReadySucceeded); diff --git a/src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs b/src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs new file mode 100644 index 00000000..de3db8db --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs @@ -0,0 +1,38 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace StellaOps.Feedser.WebService.Extensions; + +public static class ConfigurationExtensions +{ + public static IConfigurationBuilder AddFeedserYaml(this IConfigurationBuilder builder, string path) + { + if (builder is null) + { + throw new ArgumentNullException(nameof(builder)); + } + + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + return builder; + } + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + using var reader = File.OpenText(path); + var yamlObject = deserializer.Deserialize(reader); + if (yamlObject is null) + { + return builder; + } + + var json = JsonSerializer.Serialize(yamlObject); + var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + return builder.AddJsonStream(stream); + } +} diff --git a/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs b/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs new file mode 100644 index 00000000..3b1bfd39 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.WebService.Extensions; + +internal static class JobRegistrationExtensions +{ + private sealed record BuiltInJob( + string Kind, + string JobType, + string AssemblyName, + TimeSpan Timeout, + TimeSpan LeaseDuration, + string? CronExpression = null); + + private static readonly IReadOnlyList BuiltInJobs = new List + { + new("source:redhat:fetch", "StellaOps.Feedser.Source.Distro.RedHat.RedHatFetchJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "0,15,30,45 * * * *"), + new("source:redhat:parse", "StellaOps.Feedser.Source.Distro.RedHat.RedHatParseJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "5,20,35,50 * * * *"), + new("source:redhat:map", "StellaOps.Feedser.Source.Distro.RedHat.RedHatMapJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "10,25,40,55 * * * *"), + + new("source:cert-in:fetch", "StellaOps.Feedser.Source.CertIn.CertInFetchJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-in:parse", "StellaOps.Feedser.Source.CertIn.CertInParseJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-in:map", "StellaOps.Feedser.Source.CertIn.CertInMapJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:cert-fr:fetch", "StellaOps.Feedser.Source.CertFr.CertFrFetchJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-fr:parse", "StellaOps.Feedser.Source.CertFr.CertFrParseJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-fr:map", "StellaOps.Feedser.Source.CertFr.CertFrMapJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:jvn:fetch", "StellaOps.Feedser.Source.Jvn.JvnFetchJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:jvn:parse", "StellaOps.Feedser.Source.Jvn.JvnParseJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:jvn:map", "StellaOps.Feedser.Source.Jvn.JvnMapJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:ics-kaspersky:fetch", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyFetchJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:ics-kaspersky:parse", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyParseJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:ics-kaspersky:map", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyMapJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:osv:fetch", "StellaOps.Feedser.Source.Osv.OsvFetchJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:osv:parse", "StellaOps.Feedser.Source.Osv.OsvParseJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:osv:map", "StellaOps.Feedser.Source.Osv.OsvMapJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:vmware:fetch", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareFetchJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vmware:parse", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareParseJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vmware:map", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareMapJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:vndr-oracle:fetch", "StellaOps.Feedser.Source.Vndr.Oracle.OracleFetchJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vndr-oracle:parse", "StellaOps.Feedser.Source.Vndr.Oracle.OracleParseJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vndr-oracle:map", "StellaOps.Feedser.Source.Vndr.Oracle.OracleMapJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("export:json", "StellaOps.Feedser.Exporter.Json.JsonExportJob", "StellaOps.Feedser.Exporter.Json", TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(5)), + new("export:trivy-db", "StellaOps.Feedser.Exporter.TrivyDb.TrivyDbExportJob", "StellaOps.Feedser.Exporter.TrivyDb", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(10)) + }; + + public static IServiceCollection AddBuiltInFeedserJobs(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.PostConfigure(options => + { + foreach (var registration in BuiltInJobs) + { + if (options.Definitions.ContainsKey(registration.Kind)) + { + continue; + } + + var jobType = Type.GetType( + $"{registration.JobType}, {registration.AssemblyName}", + throwOnError: false, + ignoreCase: false); + + if (jobType is null) + { + continue; + } + + var timeout = registration.Timeout > TimeSpan.Zero ? registration.Timeout : options.DefaultTimeout; + var lease = registration.LeaseDuration > TimeSpan.Zero ? registration.LeaseDuration : options.DefaultLeaseDuration; + + options.Definitions[registration.Kind] = new JobDefinition( + registration.Kind, + jobType, + timeout, + lease, + registration.CronExpression, + Enabled: true); + } + }); + + return services; + } +} diff --git a/src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs b/src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs new file mode 100644 index 00000000..ae9a9366 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs @@ -0,0 +1,217 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Source.Common.Telemetry; +using StellaOps.Feedser.WebService.Diagnostics; +using StellaOps.Feedser.WebService.Options; + +namespace StellaOps.Feedser.WebService.Extensions; + +public static class TelemetryExtensions +{ + public static void ConfigureFeedserTelemetry(this WebApplicationBuilder builder, FeedserOptions options) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(options); + + var telemetry = options.Telemetry ?? new FeedserOptions.TelemetryOptions(); + + if (telemetry.EnableLogging) + { + builder.Host.UseSerilog((context, services, configuration) => + { + ConfigureSerilog(configuration, telemetry, builder.Environment.EnvironmentName, builder.Environment.ApplicationName); + }); + } + + if (!telemetry.Enabled || (!telemetry.EnableTracing && !telemetry.EnableMetrics)) + { + return; + } + + var openTelemetry = builder.Services.AddOpenTelemetry(); + + openTelemetry.ConfigureResource(resource => + { + var serviceName = telemetry.ServiceName ?? builder.Environment.ApplicationName; + var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown"; + + resource.AddService(serviceName, serviceVersion: version, serviceInstanceId: Environment.MachineName); + resource.AddAttributes(new[] + { + new KeyValuePair("deployment.environment", builder.Environment.EnvironmentName), + }); + + foreach (var attribute in telemetry.ResourceAttributes) + { + if (string.IsNullOrWhiteSpace(attribute.Key) || attribute.Value is null) + { + continue; + } + + resource.AddAttributes(new[] { new KeyValuePair(attribute.Key, attribute.Value) }); + } + }); + + if (telemetry.EnableTracing) + { + openTelemetry.WithTracing(tracing => + { + tracing + .AddSource(JobDiagnostics.ActivitySourceName) + .AddSource(SourceDiagnostics.ActivitySourceName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation(); + + ConfigureExporters(telemetry, tracing); + }); + } + + if (telemetry.EnableMetrics) + { + openTelemetry.WithMetrics(metrics => + { + metrics + .AddMeter(JobDiagnostics.MeterName) + .AddMeter(SourceDiagnostics.MeterName) + .AddMeter("StellaOps.Feedser.Source.Nvd") + .AddMeter("StellaOps.Feedser.Source.Vndr.Chromium") + .AddMeter("StellaOps.Feedser.Source.Vndr.Adobe") + .AddMeter(JobMetrics.MeterName) + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddRuntimeInstrumentation(); + + ConfigureExporters(telemetry, metrics); + }); + } + } + + private static void ConfigureSerilog(LoggerConfiguration configuration, FeedserOptions.TelemetryOptions telemetry, string environmentName, string applicationName) + { + if (!Enum.TryParse(telemetry.MinimumLogLevel, ignoreCase: true, out LogEventLevel level)) + { + level = LogEventLevel.Information; + } + + configuration + .MinimumLevel.Is(level) + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information) + .Enrich.FromLogContext() + .Enrich.With() + .Enrich.WithProperty("service.name", telemetry.ServiceName ?? applicationName) + .Enrich.WithProperty("deployment.environment", environmentName) + .WriteTo.Console(outputTemplate: "[{Timestamp:O}] [{Level:u3}] {Message:lj} {Properties}{NewLine}{Exception}"); + } + + private static void ConfigureExporters(FeedserOptions.TelemetryOptions telemetry, TracerProviderBuilder tracing) + { + if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) + { + if (telemetry.ExportConsole) + { + tracing.AddConsoleExporter(); + } + + return; + } + + tracing.AddOtlpExporter(options => + { + options.Endpoint = new Uri(telemetry.OtlpEndpoint); + var headers = BuildHeaders(telemetry); + if (!string.IsNullOrEmpty(headers)) + { + options.Headers = headers; + } + }); + + if (telemetry.ExportConsole) + { + tracing.AddConsoleExporter(); + } + } + + private static void ConfigureExporters(FeedserOptions.TelemetryOptions telemetry, MeterProviderBuilder metrics) + { + if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) + { + if (telemetry.ExportConsole) + { + metrics.AddConsoleExporter(); + } + + return; + } + + metrics.AddOtlpExporter(options => + { + options.Endpoint = new Uri(telemetry.OtlpEndpoint); + var headers = BuildHeaders(telemetry); + if (!string.IsNullOrEmpty(headers)) + { + options.Headers = headers; + } + }); + + if (telemetry.ExportConsole) + { + metrics.AddConsoleExporter(); + } + } + + private static string? BuildHeaders(FeedserOptions.TelemetryOptions telemetry) + { + if (telemetry.OtlpHeaders.Count == 0) + { + return null; + } + + return string.Join(",", telemetry.OtlpHeaders + .Where(static kvp => !string.IsNullOrWhiteSpace(kvp.Key) && !string.IsNullOrWhiteSpace(kvp.Value)) + .Select(static kvp => $"{kvp.Key}={kvp.Value}")); + } +} + +internal sealed class ActivityEnricher : ILogEventEnricher +{ + public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory) + { + var activity = Activity.Current; + if (activity is null) + { + return; + } + + if (activity.TraceId != default) + { + logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_id", activity.TraceId.ToString())); + } + + if (activity.SpanId != default) + { + logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("span_id", activity.SpanId.ToString())); + } + + if (activity.ParentSpanId != default) + { + logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("parent_span_id", activity.ParentSpanId.ToString())); + } + + if (!string.IsNullOrEmpty(activity.TraceStateString)) + { + logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty("trace_state", activity.TraceStateString)); + } + } +} diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs b/src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs new file mode 100644 index 00000000..a3bc99c5 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs @@ -0,0 +1,23 @@ +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.WebService.Jobs; + +public sealed record JobDefinitionResponse( + string Kind, + bool Enabled, + string? CronExpression, + TimeSpan Timeout, + TimeSpan LeaseDuration, + JobRunResponse? LastRun) +{ + public static JobDefinitionResponse FromDefinition(JobDefinition definition, JobRunSnapshot? lastRun) + { + return new JobDefinitionResponse( + definition.Kind, + definition.Enabled, + definition.CronExpression, + definition.Timeout, + definition.LeaseDuration, + lastRun is null ? null : JobRunResponse.FromSnapshot(lastRun)); + } +} diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs b/src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs new file mode 100644 index 00000000..285f8be7 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs @@ -0,0 +1,29 @@ +using StellaOps.Feedser.Core.Jobs; + +namespace StellaOps.Feedser.WebService.Jobs; + +public sealed record JobRunResponse( + Guid RunId, + string Kind, + JobRunStatus Status, + string Trigger, + DateTimeOffset CreatedAt, + DateTimeOffset? StartedAt, + DateTimeOffset? CompletedAt, + string? Error, + TimeSpan? Duration, + IReadOnlyDictionary Parameters) +{ + public static JobRunResponse FromSnapshot(JobRunSnapshot snapshot) + => new( + snapshot.RunId, + snapshot.Kind, + snapshot.Status, + snapshot.Trigger, + snapshot.CreatedAt, + snapshot.StartedAt, + snapshot.CompletedAt, + snapshot.Error, + snapshot.Duration, + snapshot.Parameters); +} diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs b/src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs new file mode 100644 index 00000000..a9f3c602 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Feedser.WebService.Jobs; + +public sealed class JobTriggerRequest +{ + public string Trigger { get; set; } = "api"; + + public Dictionary Parameters { get; set; } = new(StringComparer.Ordinal); +} diff --git a/src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs b/src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs new file mode 100644 index 00000000..59a31fec --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs @@ -0,0 +1,53 @@ +namespace StellaOps.Feedser.WebService.Options; + +public sealed class FeedserOptions +{ + public StorageOptions Storage { get; set; } = new(); + + public PluginOptions Plugins { get; set; } = new(); + + public TelemetryOptions Telemetry { get; set; } = new(); + + public sealed class StorageOptions + { + public string Driver { get; set; } = "mongo"; + + public string Dsn { get; set; } = string.Empty; + + public string? Database { get; set; } + + public int CommandTimeoutSeconds { get; set; } = 30; + } + + public sealed class PluginOptions + { + public string? BaseDirectory { get; set; } + + public string? Directory { get; set; } + + public IList SearchPatterns { get; set; } = new List(); + } + + public sealed class TelemetryOptions + { + public bool Enabled { get; set; } = true; + + public bool EnableTracing { get; set; } = true; + + public bool EnableMetrics { get; set; } = true; + + public bool EnableLogging { get; set; } = true; + + public string MinimumLogLevel { get; set; } = "Information"; + + public string? ServiceName { get; set; } + + public string? OtlpEndpoint { get; set; } + + public IDictionary OtlpHeaders { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public IDictionary ResourceAttributes { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public bool ExportConsole { get; set; } + } +} diff --git a/src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs b/src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs new file mode 100644 index 00000000..95d5a74b --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Logging; + +namespace StellaOps.Feedser.WebService.Options; + +public static class FeedserOptionsValidator +{ + public static void Validate(FeedserOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + if (!string.Equals(options.Storage.Driver, "mongo", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Only Mongo storage driver is supported (storage.driver == 'mongo')."); + } + + if (string.IsNullOrWhiteSpace(options.Storage.Dsn)) + { + throw new InvalidOperationException("Storage DSN must be configured."); + } + + if (options.Storage.CommandTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Command timeout must be greater than zero seconds."); + } + + options.Telemetry ??= new FeedserOptions.TelemetryOptions(); + + if (!Enum.TryParse(options.Telemetry.MinimumLogLevel, ignoreCase: true, out LogLevel _)) + { + throw new InvalidOperationException($"Telemetry minimum log level '{options.Telemetry.MinimumLogLevel}' is invalid."); + } + + if (!string.IsNullOrWhiteSpace(options.Telemetry.OtlpEndpoint) && !Uri.TryCreate(options.Telemetry.OtlpEndpoint, UriKind.Absolute, out _)) + { + throw new InvalidOperationException("Telemetry OTLP endpoint must be an absolute URI."); + } + + foreach (var attribute in options.Telemetry.ResourceAttributes) + { + if (string.IsNullOrWhiteSpace(attribute.Key)) + { + throw new InvalidOperationException("Telemetry resource attribute keys must be non-empty."); + } + } + + foreach (var header in options.Telemetry.OtlpHeaders) + { + if (string.IsNullOrWhiteSpace(header.Key)) + { + throw new InvalidOperationException("Telemetry OTLP header names must be non-empty."); + } + } + } +} diff --git a/src/StellaOps.Feedser.WebService/Program.cs b/src/StellaOps.Feedser.WebService/Program.cs new file mode 100644 index 00000000..7a47bf56 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Program.cs @@ -0,0 +1,472 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.AspNetCore.Diagnostics; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Feedser.Core.Jobs; +using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Feedser.WebService.Diagnostics; +using Serilog; +using StellaOps.Feedser.WebService.Extensions; +using StellaOps.Feedser.WebService.Jobs; +using StellaOps.Feedser.WebService.Options; +using Serilog.Events; +using StellaOps.Plugin.DependencyInjection; +using StellaOps.Plugin.Hosting; + +var builder = WebApplication.CreateBuilder(args); + +builder.Configuration + .AddJsonFile("appsettings.json", optional: true, reloadOnChange: false) + .AddFeedserYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/feedser.yaml")) + .AddEnvironmentVariables(prefix: "FEEDSER_"); + +builder.Services.AddOptions() + .Bind(builder.Configuration) + .PostConfigure(FeedserOptionsValidator.Validate) + .ValidateOnStart(); + +var optionsSnapshot = new FeedserOptions(); +builder.Configuration.Bind(optionsSnapshot); +FeedserOptionsValidator.Validate(optionsSnapshot); + +builder.ConfigureFeedserTelemetry(optionsSnapshot); + +builder.Services.AddMongoStorage(storageOptions => +{ + storageOptions.ConnectionString = optionsSnapshot.Storage.Dsn; + storageOptions.DatabaseName = optionsSnapshot.Storage.Database; + storageOptions.CommandTimeout = TimeSpan.FromSeconds(optionsSnapshot.Storage.CommandTimeoutSeconds); +}); + +builder.Services.AddJobScheduler(); +builder.Services.AddBuiltInFeedserJobs(); + +builder.Services.AddSingleton(sp => new ServiceStatus(sp.GetRequiredService())); + +var pluginHostOptions = BuildPluginOptions(optionsSnapshot, builder.Environment.ContentRootPath); +builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions); + +builder.Services.AddEndpointsApiExplorer(); + +var app = builder.Build(); + +var jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web); +jsonOptions.Converters.Add(new JsonStringEnumConverter()); + +app.UseSerilogRequestLogging(options => +{ + options.IncludeQueryInRequestPath = true; + options.GetLevel = (httpContext, elapsedMs, exception) => exception is null ? LogEventLevel.Information : LogEventLevel.Error; + options.EnrichDiagnosticContext = (diagnosticContext, httpContext) => + { + diagnosticContext.Set("RequestId", httpContext.TraceIdentifier); + diagnosticContext.Set("UserAgent", httpContext.Request.Headers.UserAgent.ToString()); + if (Activity.Current is { TraceId: var traceId } && traceId != default) + { + diagnosticContext.Set("TraceId", traceId.ToString()); + } + }; +}); + +app.UseExceptionHandler(errorApp => +{ + errorApp.Run(async context => + { + context.Response.ContentType = "application/problem+json"; + var feature = context.Features.Get(); + var error = feature?.Error; + + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["traceId"] = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier, + }; + + var problem = Results.Problem( + detail: error?.Message, + instance: context.Request.Path, + statusCode: StatusCodes.Status500InternalServerError, + title: "Unexpected server error", + type: ProblemTypes.JobFailure, + extensions: extensions); + + await problem.ExecuteAsync(context); + }); +}); + +IResult JsonResult(T value, int? statusCode = null) +{ + var payload = JsonSerializer.Serialize(value, jsonOptions); + return Results.Content(payload, "application/json", Encoding.UTF8, statusCode); +} + +IResult Problem(HttpContext context, string title, int statusCode, string type, string? detail = null, IDictionary? extensions = null) +{ + var traceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier; + extensions ??= new Dictionary(StringComparer.Ordinal) + { + ["traceId"] = traceId, + }; + + if (!extensions.ContainsKey("traceId")) + { + extensions["traceId"] = traceId; + } + + return Results.Problem( + detail: detail, + instance: context.Request.Path, + statusCode: statusCode, + title: title, + type: type, + extensions: extensions); +} + +static KeyValuePair[] BuildJobMetricTags(string jobKind, string trigger, string outcome) + => new[] + { + new KeyValuePair("job.kind", jobKind), + new KeyValuePair("job.trigger", trigger), + new KeyValuePair("job.outcome", outcome), + }; + +void ApplyNoCache(HttpResponse response) +{ + if (response is null) + { + return; + } + + response.Headers.CacheControl = "no-store, no-cache, max-age=0, must-revalidate"; + response.Headers.Pragma = "no-cache"; + response.Headers["Expires"] = "0"; +} + +await InitializeMongoAsync(app); + +app.MapGet("/", () => Results.Ok(new { message = "StellaOps Feedser" })); + +app.MapGet("/health", (IOptions opts, ServiceStatus status, HttpContext context) => +{ + ApplyNoCache(context.Response); + + var snapshot = status.CreateSnapshot(); + var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d); + + var storage = new StorageBootstrapHealth( + Driver: opts.Value.Storage.Driver, + Completed: snapshot.BootstrapCompletedAt is not null, + CompletedAt: snapshot.BootstrapCompletedAt, + DurationMs: snapshot.BootstrapDuration?.TotalMilliseconds); + + var telemetry = new TelemetryHealth( + Enabled: opts.Value.Telemetry.Enabled, + Tracing: opts.Value.Telemetry.EnableTracing, + Metrics: opts.Value.Telemetry.EnableMetrics, + Logging: opts.Value.Telemetry.EnableLogging); + + var response = new HealthDocument( + Status: "healthy", + StartedAt: snapshot.StartedAt, + UptimeSeconds: uptimeSeconds, + Storage: storage, + Telemetry: telemetry); + + return JsonResult(response); +}); + +app.MapGet("/ready", async (IMongoDatabase database, ServiceStatus status, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var stopwatch = Stopwatch.StartNew(); + try + { + await database.RunCommandAsync((Command)"{ ping: 1 }", cancellationToken: cancellationToken).ConfigureAwait(false); + stopwatch.Stop(); + status.RecordMongoCheck(success: true, latency: stopwatch.Elapsed, error: null); + + var snapshot = status.CreateSnapshot(); + var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d); + + var mongo = new MongoReadyHealth( + Status: "ready", + LatencyMs: snapshot.LastMongoLatency?.TotalMilliseconds, + CheckedAt: snapshot.LastReadyCheckAt, + Error: null); + + var response = new ReadyDocument( + Status: "ready", + StartedAt: snapshot.StartedAt, + UptimeSeconds: uptimeSeconds, + Mongo: mongo); + + return JsonResult(response); + } + catch (Exception ex) + { + stopwatch.Stop(); + status.RecordMongoCheck(success: false, latency: stopwatch.Elapsed, error: ex.Message); + + var snapshot = status.CreateSnapshot(); + var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d); + + var mongo = new MongoReadyHealth( + Status: "unready", + LatencyMs: snapshot.LastMongoLatency?.TotalMilliseconds, + CheckedAt: snapshot.LastReadyCheckAt, + Error: snapshot.LastMongoError ?? ex.Message); + + var response = new ReadyDocument( + Status: "unready", + StartedAt: snapshot.StartedAt, + UptimeSeconds: uptimeSeconds, + Mongo: mongo); + + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["mongoLatencyMs"] = snapshot.LastMongoLatency?.TotalMilliseconds, + ["mongoError"] = snapshot.LastMongoError ?? ex.Message, + }; + + return Problem(context, "Mongo unavailable", StatusCodes.Status503ServiceUnavailable, ProblemTypes.ServiceUnavailable, snapshot.LastMongoError ?? ex.Message, extensions); + } +}); + +app.MapGet("/jobs", async (string? kind, int? limit, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200); + var runs = await coordinator.GetRecentRunsAsync(kind, take, cancellationToken).ConfigureAwait(false); + var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); + return JsonResult(payload); +}); + +app.MapGet("/jobs/{runId:guid}", async (Guid runId, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var run = await coordinator.GetRunAsync(runId, cancellationToken).ConfigureAwait(false); + if (run is null) + { + return Problem(context, "Job run not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job run '{runId}' was not found."); + } + + return JsonResult(JobRunResponse.FromSnapshot(run)); +}); + +app.MapGet("/jobs/definitions", async (IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var definitions = await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false); + if (definitions.Count == 0) + { + return JsonResult(Array.Empty()); + } + + var definitionKinds = definitions.Select(static definition => definition.Kind).ToArray(); + var lastRuns = await coordinator.GetLastRunsAsync(definitionKinds, cancellationToken).ConfigureAwait(false); + + var responses = new List(definitions.Count); + foreach (var definition in definitions) + { + lastRuns.TryGetValue(definition.Kind, out var lastRun); + responses.Add(JobDefinitionResponse.FromDefinition(definition, lastRun)); + } + + return JsonResult(responses); +}); + +app.MapGet("/jobs/definitions/{kind}", async (string kind, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false)) + .FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal)); + + if (definition is null) + { + return Problem(context, "Job definition not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job kind '{kind}' is not registered."); + } + + var lastRuns = await coordinator.GetLastRunsAsync(new[] { definition.Kind }, cancellationToken).ConfigureAwait(false); + lastRuns.TryGetValue(definition.Kind, out var lastRun); + + var response = JobDefinitionResponse.FromDefinition(definition, lastRun); + return JsonResult(response); +}); + +app.MapGet("/jobs/definitions/{kind}/runs", async (string kind, int? limit, IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false)) + .FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal)); + + if (definition is null) + { + return Problem(context, "Job definition not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, $"Job kind '{kind}' is not registered."); + } + + var take = Math.Clamp(limit.GetValueOrDefault(20), 1, 200); + var runs = await coordinator.GetRecentRunsAsync(kind, take, cancellationToken).ConfigureAwait(false); + var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); + return JsonResult(payload); +}); + +app.MapGet("/jobs/active", async (IJobCoordinator coordinator, HttpContext context, CancellationToken cancellationToken) => +{ + ApplyNoCache(context.Response); + + var runs = await coordinator.GetActiveRunsAsync(cancellationToken).ConfigureAwait(false); + var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); + return JsonResult(payload); +}); + +app.MapPost("/jobs/{*jobKind}", async (string jobKind, JobTriggerRequest request, IJobCoordinator coordinator, HttpContext context) => +{ + ApplyNoCache(context.Response); + + request ??= new JobTriggerRequest(); + request.Parameters ??= new Dictionary(StringComparer.Ordinal); + var trigger = string.IsNullOrWhiteSpace(request.Trigger) ? "api" : request.Trigger; + + var lifetime = context.RequestServices.GetRequiredService(); + var result = await coordinator.TriggerAsync(jobKind, request.Parameters, trigger, lifetime.ApplicationStopping).ConfigureAwait(false); + + var outcome = result.Outcome; + var tags = BuildJobMetricTags(jobKind, trigger, outcome.ToString().ToLowerInvariant()); + + switch (outcome) + { + case JobTriggerOutcome.Accepted: + JobMetrics.TriggerCounter.Add(1, tags); + if (result.Run is null) + { + return Results.Accepted(); + } + + var acceptedRun = JobRunResponse.FromSnapshot(result.Run); + return Results.Accepted($"/jobs/{acceptedRun.RunId}", acceptedRun); + + case JobTriggerOutcome.NotFound: + JobMetrics.TriggerConflictCounter.Add(1, tags); + return Problem(context, "Job not found", StatusCodes.Status404NotFound, ProblemTypes.NotFound, result.ErrorMessage ?? $"Job '{jobKind}' is not registered."); + + case JobTriggerOutcome.Disabled: + JobMetrics.TriggerConflictCounter.Add(1, tags); + return Problem(context, "Job disabled", StatusCodes.Status423Locked, ProblemTypes.Locked, result.ErrorMessage ?? $"Job '{jobKind}' is disabled."); + + case JobTriggerOutcome.AlreadyRunning: + JobMetrics.TriggerConflictCounter.Add(1, tags); + return Problem(context, "Job already running", StatusCodes.Status409Conflict, ProblemTypes.Conflict, result.ErrorMessage ?? $"Job '{jobKind}' already has an active run."); + + case JobTriggerOutcome.LeaseRejected: + JobMetrics.TriggerConflictCounter.Add(1, tags); + return Problem(context, "Job lease rejected", StatusCodes.Status409Conflict, ProblemTypes.LeaseRejected, result.ErrorMessage ?? $"Job '{jobKind}' could not acquire a lease."); + + case JobTriggerOutcome.InvalidParameters: + { + JobMetrics.TriggerConflictCounter.Add(1, tags); + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["parameters"] = request.Parameters, + }; + return Problem(context, "Invalid job parameters", StatusCodes.Status400BadRequest, ProblemTypes.Validation, result.ErrorMessage, extensions); + } + + case JobTriggerOutcome.Cancelled: + { + JobMetrics.TriggerConflictCounter.Add(1, tags); + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["run"] = result.Run is null ? null : JobRunResponse.FromSnapshot(result.Run), + }; + + return Problem(context, "Job cancelled", StatusCodes.Status409Conflict, ProblemTypes.Conflict, result.ErrorMessage ?? $"Job '{jobKind}' was cancelled before completion.", extensions); + } + + case JobTriggerOutcome.Failed: + { + JobMetrics.TriggerFailureCounter.Add(1, tags); + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["run"] = result.Run is null ? null : JobRunResponse.FromSnapshot(result.Run), + }; + + return Problem(context, "Job execution failed", StatusCodes.Status500InternalServerError, ProblemTypes.JobFailure, result.ErrorMessage, extensions); + } + + default: + JobMetrics.TriggerFailureCounter.Add(1, tags); + return Problem(context, "Unexpected job outcome", StatusCodes.Status500InternalServerError, ProblemTypes.JobFailure, $"Job '{jobKind}' returned outcome '{outcome}'."); + } +}); + +await app.RunAsync(); + +static PluginHostOptions BuildPluginOptions(FeedserOptions options, string contentRoot) +{ + var pluginOptions = new PluginHostOptions + { + BaseDirectory = options.Plugins.BaseDirectory ?? contentRoot, + PluginsDirectory = options.Plugins.Directory ?? Path.Combine(contentRoot, "PluginBinaries"), + EnsureDirectoryExists = true, + RecursiveSearch = false, + }; + + if (options.Plugins.SearchPatterns.Count == 0) + { + pluginOptions.SearchPatterns.Add("StellaOps.Feedser.Plugin.*.dll"); + } + else + { + foreach (var pattern in options.Plugins.SearchPatterns) + { + if (!string.IsNullOrWhiteSpace(pattern)) + { + pluginOptions.SearchPatterns.Add(pattern); + } + } + } + + return pluginOptions; +} + +static async Task InitializeMongoAsync(WebApplication app) +{ + await using var scope = app.Services.CreateAsyncScope(); + var bootstrapper = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("MongoBootstrapper"); + var status = scope.ServiceProvider.GetRequiredService(); + + var stopwatch = Stopwatch.StartNew(); + + try + { + await bootstrapper.InitializeAsync(app.Lifetime.ApplicationStopping).ConfigureAwait(false); + stopwatch.Stop(); + status.MarkBootstrapCompleted(stopwatch.Elapsed); + logger.LogInformation("Mongo bootstrap completed in {ElapsedMs} ms", stopwatch.Elapsed.TotalMilliseconds); + } + catch (Exception ex) + { + stopwatch.Stop(); + status.RecordMongoCheck(success: false, latency: stopwatch.Elapsed, error: ex.Message); + logger.LogCritical(ex, "Mongo bootstrap failed after {ElapsedMs} ms", stopwatch.Elapsed.TotalMilliseconds); + throw; + } +} + +public partial class Program; diff --git a/src/StellaOps.Feedser.WebService/Properties/launchSettings.json b/src/StellaOps.Feedser.WebService/Properties/launchSettings.json new file mode 100644 index 00000000..14722a5f --- /dev/null +++ b/src/StellaOps.Feedser.WebService/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "StellaOps.Feedser.WebService": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:50411;http://localhost:50412" + } + } +} \ No newline at end of file diff --git a/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj b/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj new file mode 100644 index 00000000..cc8f5866 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj @@ -0,0 +1,31 @@ + + + net10.0 + preview + enable + enable + true + StellaOps.Feedser.WebService + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.WebService/TASKS.md b/src/StellaOps.Feedser.WebService/TASKS.md new file mode 100644 index 00000000..01016940 --- /dev/null +++ b/src/StellaOps.Feedser.WebService/TASKS.md @@ -0,0 +1,16 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Bind & validate FeedserOptions|BE-Base|WebService|DONE – options bound/validated with failure logging.| +|Mongo service wiring|BE-Base|Storage.Mongo|DONE – wiring delegated to `AddMongoStorage`.| +|Bootstrapper execution on start|BE-Base|Storage.Mongo|DONE – startup calls `MongoBootstrapper.InitializeAsync`.| +|Plugin host options finalization|BE-Base|Plugins|DONE – default plugin directories/search patterns configured.| +|Jobs API contract tests|QA|Core|DONE – WebServiceEndpointsTests now cover success payloads, filtering, and trigger outcome mapping.| +|Health/Ready probes|DevOps|Ops|DONE – `/health` and `/ready` endpoints implemented.| +|Serilog + OTEL integration hooks|BE-Base|Observability|DONE – `TelemetryExtensions` wires Serilog + OTEL with configurable exporters.| +|Register built-in jobs (sources/exporters)|BE-Base|Core|DONE – AddBuiltInFeedserJobs adds fallback scheduler definitions for core connectors and exporters via reflection.| +|HTTP problem details consistency|BE-Base|WebService|DONE – API errors now emit RFC7807 responses with trace identifiers and typed problem categories.| +|Request logging and metrics|BE-Base|Observability|DONE – Serilog request logging enabled with enriched context and web.jobs counters published via OpenTelemetry.| +|Endpoint smoke tests (health/ready/jobs error paths)|QA|WebService|DONE – WebServiceEndpointsTests assert success and problem responses for health, ready, and job trigger error paths.| +|Batch job definition last-run lookup|BE-Base|Core|DONE – definitions endpoint now precomputes kinds array and reuses batched last-run dictionary; manual smoke verified via local GET `/jobs/definitions`.| +|Add no-cache headers to health/readiness/jobs APIs|BE-Base|WebService|DONE – helper applies Cache-Control/Pragma/Expires on all health/ready/jobs endpoints; awaiting automated probe tests once connector fixtures stabilize.| diff --git a/src/StellaOps.Feedser.sln b/src/StellaOps.Feedser.sln new file mode 100644 index 00000000..c542ef93 --- /dev/null +++ b/src/StellaOps.Feedser.sln @@ -0,0 +1,860 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Acsc", "StellaOps.Feedser.Source.Acsc\StellaOps.Feedser.Source.Acsc.csproj", "{CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common", "StellaOps.Feedser.Source.Common\StellaOps.Feedser.Source.Common.csproj", "{E9DE840D-0760-4324-98E2-7F2CBE06DC1A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models", "StellaOps.Feedser.Models\StellaOps.Feedser.Models.csproj", "{061B0042-9A6C-4CFD-9E48-4D3F3B924442}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Cisa", "StellaOps.Feedser.Source.Ics.Cisa\StellaOps.Feedser.Source.Ics.Cisa.csproj", "{6A301F32-2EEE-491B-9DB9-3BF26D032F07}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core", "StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj", "{AFCCC916-58E8-4676-AABB-54B04CEA3392}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo", "StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj", "{BF3DAB2F-E46E-49C1-9BA5-AA389763A632}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization", "StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj", "{429BAA6A-706D-489A-846F-4B0EF1B15121}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge", "StellaOps.Feedser.Merge\StellaOps.Feedser.Merge.csproj", "{085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json", "StellaOps.Feedser.Exporter.Json\StellaOps.Feedser.Exporter.Json.csproj", "{1C5506B8-C01B-4419-B888-A48F441E0C69}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb", "StellaOps.Feedser.Exporter.TrivyDb\StellaOps.Feedser.Exporter.TrivyDb.csproj", "{4D936BC4-5520-4642-A237-4106E97BC7A0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "StellaOps.Plugin\StellaOps.Plugin.csproj", "{B85C1C0E-B245-44FB-877E-C112DE29041A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService", "StellaOps.Feedser.WebService\StellaOps.Feedser.WebService.csproj", "{2C970A0F-FE3D-425B-B1B3-A008B194F5C2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cccs", "StellaOps.Feedser.Source.Cccs\StellaOps.Feedser.Source.Cccs.csproj", "{A7035381-6D20-4A07-817B-A324ED735EB3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian", "StellaOps.Feedser.Source.Distro.Debian\StellaOps.Feedser.Source.Distro.Debian.csproj", "{404F5F6E-37E4-4EF9-B09D-6634366B5D44}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Ubuntu", "StellaOps.Feedser.Source.Distro.Ubuntu\StellaOps.Feedser.Source.Distro.Ubuntu.csproj", "{1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kisa", "StellaOps.Feedser.Source.Kisa\StellaOps.Feedser.Source.Kisa.csproj", "{23055A20-7079-4336-AD30-EFAA2FA11665}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertCc", "StellaOps.Feedser.Source.CertCc\StellaOps.Feedser.Source.CertCc.csproj", "{C2304954-9B15-4776-8DB6-22E293D311E4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr", "StellaOps.Feedser.Source.CertFr\StellaOps.Feedser.Source.CertFr.csproj", "{E6895821-ED23-46D2-A5DC-06D61F90EC27}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd", "StellaOps.Feedser.Source.Nvd\StellaOps.Feedser.Source.Nvd.csproj", "{378CB675-D70B-4A95-B324-62B67D79AAB7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle", "StellaOps.Feedser.Source.Vndr.Oracle\StellaOps.Feedser.Source.Vndr.Oracle.csproj", "{53AD2E55-B0F5-46AD-BFE5-82F486371872}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Nkcki", "StellaOps.Feedser.Source.Ru.Nkcki\StellaOps.Feedser.Source.Ru.Nkcki.csproj", "{B880C99C-C0BD-4953-95AD-2C76BC43F760}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Suse", "StellaOps.Feedser.Source.Distro.Suse\StellaOps.Feedser.Source.Distro.Suse.csproj", "{23422F67-C1FB-4FF4-899C-706BCD63D9FD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Bdu", "StellaOps.Feedser.Source.Ru.Bdu\StellaOps.Feedser.Source.Ru.Bdu.csproj", "{16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kev", "StellaOps.Feedser.Source.Kev\StellaOps.Feedser.Source.Kev.csproj", "{20DB9837-715B-4515-98C6-14B50060B765}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky", "StellaOps.Feedser.Source.Ics.Kaspersky\StellaOps.Feedser.Source.Ics.Kaspersky.csproj", "{10849EE2-9F34-4C23-BBB4-916A59CDB7F4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv", "StellaOps.Feedser.Source.Osv\StellaOps.Feedser.Source.Osv.csproj", "{EFB16EDB-78D4-4601-852E-F4B37655FA13}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn", "StellaOps.Feedser.Source.Jvn\StellaOps.Feedser.Source.Jvn.csproj", "{02289F61-0173-42CC-B8F2-25CC53F8E066}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertBund", "StellaOps.Feedser.Source.CertBund\StellaOps.Feedser.Source.CertBund.csproj", "{4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cve", "StellaOps.Feedser.Source.Cve\StellaOps.Feedser.Source.Cve.csproj", "{EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Cisco", "StellaOps.Feedser.Source.Vndr.Cisco\StellaOps.Feedser.Source.Vndr.Cisco.csproj", "{19957518-A422-4622-9FD1-621DF3E31869}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Msrc", "StellaOps.Feedser.Source.Vndr.Msrc\StellaOps.Feedser.Source.Vndr.Msrc.csproj", "{69C4C061-F5A0-4EAA-A4CD-9A513523952A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium", "StellaOps.Feedser.Source.Vndr.Chromium\StellaOps.Feedser.Source.Vndr.Chromium.csproj", "{C7F7DE6F-A369-4F43-9864-286DCEC615F8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Apple", "StellaOps.Feedser.Source.Vndr.Apple\StellaOps.Feedser.Source.Vndr.Apple.csproj", "{1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware", "StellaOps.Feedser.Source.Vndr.Vmware\StellaOps.Feedser.Source.Vndr.Vmware.csproj", "{7255C38D-5A16-4A4D-98CE-CF0FD516B68E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe", "StellaOps.Feedser.Source.Vndr.Adobe\StellaOps.Feedser.Source.Vndr.Adobe.csproj", "{C3A42AA3-800D-4398-A077-5560EE6451EF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn", "StellaOps.Feedser.Source.CertIn\StellaOps.Feedser.Source.CertIn.csproj", "{5016963A-6FC9-4063-AB83-2D1F9A2BC627}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ghsa", "StellaOps.Feedser.Source.Ghsa\StellaOps.Feedser.Source.Ghsa.csproj", "{72F43F43-F852-487F-8334-91D438CE2F7C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat", "StellaOps.Feedser.Source.Distro.RedHat\StellaOps.Feedser.Source.Distro.RedHat.csproj", "{A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{F622D38D-DA49-473E-B724-E706F8113CF2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core.Tests", "StellaOps.Feedser.Core.Tests\StellaOps.Feedser.Core.Tests.csproj", "{3A3D7610-C864-4413-B07E-9E8C2A49A90E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge.Tests", "StellaOps.Feedser.Merge.Tests\StellaOps.Feedser.Merge.Tests.csproj", "{9C4DEE96-CD7D-4AE3-A811-0B48B477003B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models.Tests", "StellaOps.Feedser.Models.Tests\StellaOps.Feedser.Models.Tests.csproj", "{437B2667-9461-47D2-B75B-4D2E03D69B94}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization.Tests", "StellaOps.Feedser.Normalization.Tests\StellaOps.Feedser.Normalization.Tests.csproj", "{8249DF28-CDAF-4DEF-A912-C27F57B67FD5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo.Tests", "StellaOps.Feedser.Storage.Mongo.Tests\StellaOps.Feedser.Storage.Mongo.Tests.csproj", "{CBFB015B-C069-475F-A476-D52222729804}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json.Tests", "StellaOps.Feedser.Exporter.Json.Tests\StellaOps.Feedser.Exporter.Json.Tests.csproj", "{2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb.Tests", "StellaOps.Feedser.Exporter.TrivyDb.Tests\StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj", "{3EB22234-642E-4533-BCC3-93E8ED443B1D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService.Tests", "StellaOps.Feedser.WebService.Tests\StellaOps.Feedser.WebService.Tests.csproj", "{84A5DE81-4444-499A-93BF-6DC4CA72F8D4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common.Tests", "StellaOps.Feedser.Source.Common.Tests\StellaOps.Feedser.Source.Common.Tests.csproj", "{42E21E1D-C3DE-4765-93E9-39391BB5C802}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd.Tests", "StellaOps.Feedser.Source.Nvd.Tests\StellaOps.Feedser.Source.Nvd.Tests.csproj", "{B6E2EE26-B297-4AB9-A47E-A227F5EAE108}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat.Tests", "StellaOps.Feedser.Source.Distro.RedHat.Tests\StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj", "{CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium.Tests", "StellaOps.Feedser.Source.Vndr.Chromium.Tests\StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj", "{2891FCDE-BB89-46F0-A40C-368EF804DB44}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe.Tests", "StellaOps.Feedser.Source.Vndr.Adobe.Tests\StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj", "{B91C60FB-926F-47C3-BFD0-6DD145308344}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle.Tests", "StellaOps.Feedser.Source.Vndr.Oracle.Tests\StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj", "{30DF89D1-D66D-4078-8A3B-951637A42265}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware.Tests", "StellaOps.Feedser.Source.Vndr.Vmware.Tests\StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj", "{6E98C770-72FF-41FA-8C42-30AABAAF5B4E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn.Tests", "StellaOps.Feedser.Source.CertIn.Tests\StellaOps.Feedser.Source.CertIn.Tests.csproj", "{79B36C92-BA93-4406-AB75-6F2282DDFF01}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr.Tests", "StellaOps.Feedser.Source.CertFr.Tests\StellaOps.Feedser.Source.CertFr.Tests.csproj", "{4B60FA53-81F6-4AB6-BE9F-DE0992E11977}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky.Tests", "StellaOps.Feedser.Source.Ics.Kaspersky.Tests\StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj", "{6BBA820B-8443-4832-91C3-3AB002006494}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn.Tests", "StellaOps.Feedser.Source.Jvn.Tests\StellaOps.Feedser.Source.Jvn.Tests.csproj", "{7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv.Tests", "StellaOps.Feedser.Source.Osv.Tests\StellaOps.Feedser.Source.Osv.Tests.csproj", "{F892BFFD-9101-4D59-B6FD-C532EB04D51F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Testing", "StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj", "{EAE910FC-188C-41C3-822A-623964CABE48}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x64.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x64.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x86.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x86.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|Any CPU.Build.0 = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x64.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x64.Build.0 = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x86.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x86.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x64.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x64.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x86.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x86.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|Any CPU.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x64.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x64.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x86.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x86.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|Any CPU.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x64.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x64.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x86.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x86.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|Any CPU.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|Any CPU.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x64.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x64.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x86.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x86.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x64.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x64.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x86.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x86.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|Any CPU.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x64.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x64.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x86.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x86.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x64.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x64.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x86.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x86.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|Any CPU.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x64.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x64.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x86.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x86.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x64.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x64.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x86.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x86.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|Any CPU.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x64.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x64.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x86.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x86.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|Any CPU.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x64.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x64.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x86.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x86.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|Any CPU.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|Any CPU.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x64.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x64.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x86.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x86.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x64.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x64.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x86.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x86.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|Any CPU.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x64.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x64.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x86.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x86.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x64.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x64.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x86.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x86.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|Any CPU.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x64.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x64.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x86.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x86.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x64.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x64.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x86.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x86.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|Any CPU.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x64.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x64.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x86.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x86.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x64.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x64.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x86.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x86.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|Any CPU.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x64.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x64.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x86.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x86.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x64.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x64.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x86.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x86.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|Any CPU.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x64.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x64.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x86.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x86.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x64.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x64.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x86.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x86.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|Any CPU.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x64.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x64.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x86.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x86.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|Any CPU.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x64.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x64.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x86.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x86.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|Any CPU.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|Any CPU.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x64.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x64.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x86.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x86.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x64.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x64.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x86.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x86.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|Any CPU.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x64.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x64.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x86.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x86.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x64.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x64.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x86.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x86.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|Any CPU.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x64.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x64.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x86.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x86.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x64.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x64.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x86.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x86.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|Any CPU.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x64.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x64.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x86.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x86.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x64.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x64.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x86.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x86.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|Any CPU.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x64.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x64.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x86.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x86.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x64.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x64.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x86.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x86.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|Any CPU.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x64.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x64.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x86.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x86.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|Any CPU.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x64.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x64.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x86.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x86.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|Any CPU.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|Any CPU.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x64.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x64.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x86.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x86.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x64.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x64.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x86.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x86.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|Any CPU.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x64.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x64.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x86.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x86.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x64.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x64.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x86.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x86.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|Any CPU.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x64.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x64.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x86.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x86.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x64.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x64.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x86.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x86.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|Any CPU.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x64.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x64.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x86.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x86.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|Any CPU.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x64.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x64.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x86.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x86.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|Any CPU.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|Any CPU.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x64.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x64.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x86.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x86.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x64.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x86.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x86.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|Any CPU.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x64.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x64.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x86.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x86.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x64.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x64.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x86.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x86.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|Any CPU.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x64.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x64.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x86.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x86.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|Any CPU.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x64.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x64.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x86.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x86.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|Any CPU.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|Any CPU.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x64.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x64.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x86.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x86.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x64.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x64.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x86.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x86.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|Any CPU.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x64.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x64.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x86.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x86.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x64.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x64.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x86.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x86.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|Any CPU.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x64.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x64.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x86.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x86.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|Any CPU.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x64.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x64.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x86.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x86.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|Any CPU.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|Any CPU.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x64.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x64.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x86.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x86.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x64.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x64.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x86.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x86.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|Any CPU.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x64.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x64.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x86.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x86.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x64.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x64.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x86.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x86.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|Any CPU.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x64.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x64.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x86.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x86.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x64.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x64.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x86.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x86.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|Any CPU.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x64.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x64.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x86.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x86.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x64.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x86.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|Any CPU.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x64.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x64.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x86.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x86.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x64.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x64.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x86.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x86.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|Any CPU.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x64.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x64.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x86.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x86.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x64.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x64.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x86.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x86.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|Any CPU.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x64.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x64.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x86.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x86.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x64.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x64.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x86.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x86.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|Any CPU.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x64.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x64.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x86.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x86.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x64.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x64.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x86.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x86.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|Any CPU.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x64.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x64.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x86.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x86.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x64.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x64.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x86.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x86.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|Any CPU.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x64.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x64.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x86.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x86.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x64.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x64.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x86.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x86.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|Any CPU.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x64.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x64.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x86.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x86.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x64.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x64.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x86.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x86.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|Any CPU.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x64.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x64.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x86.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x86.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|Any CPU.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x64.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x64.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x86.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x86.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|Any CPU.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|Any CPU.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x64.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x64.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x86.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x86.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x64.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x64.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x86.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x86.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|Any CPU.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x64.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x64.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x86.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x86.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x64.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x64.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x86.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x86.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|Any CPU.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x64.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x64.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x86.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x86.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x64.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x64.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x86.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x86.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|Any CPU.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x64.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x64.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x86.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x86.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x64.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x64.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x86.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x86.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|Any CPU.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x64.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x64.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x86.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x86.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x64.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x64.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x86.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x86.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|Any CPU.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x64.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x64.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x86.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x86.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|Any CPU.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x64.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x64.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x86.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x86.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|Any CPU.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|Any CPU.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x64.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x64.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x86.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x86.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x64.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x64.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x86.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x86.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|Any CPU.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x64.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x64.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x86.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x86.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x64.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x64.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x86.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x86.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|Any CPU.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x64.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x64.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x86.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x86.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x64.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x64.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x86.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x86.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|Any CPU.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x64.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x64.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x86.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x86.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x64.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x64.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x86.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x86.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|Any CPU.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x64.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x64.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x86.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x86.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|Any CPU.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x64.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x64.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x86.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x86.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|Any CPU.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|Any CPU.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x64.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x64.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x86.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x86.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x64.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x64.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x86.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x86.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|Any CPU.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x64.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x64.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x86.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x86.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x64.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x64.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x86.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x86.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|Any CPU.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x64.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x64.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x86.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x86.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x64.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x64.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x86.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x86.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|Any CPU.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x64.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x64.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x86.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x86.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x64.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x64.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x86.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x86.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|Any CPU.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x64.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x64.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x86.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x86.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x64.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x64.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x86.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x86.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|Any CPU.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x64.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x64.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x86.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x86.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x64.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x64.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x86.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x86.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|Any CPU.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x64.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x64.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x86.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x86.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x64.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x64.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x86.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x86.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|Any CPU.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs b/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs new file mode 100644 index 00000000..5afd955a --- /dev/null +++ b/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs @@ -0,0 +1,91 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.DependencyInjection; +using StellaOps.Plugin.Hosting; +using StellaOps.Plugin.Internal; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Plugin.DependencyInjection; + +public static class PluginDependencyInjectionExtensions +{ + public static IServiceCollection RegisterPluginRoutines( + this IServiceCollection services, + IConfiguration configuration, + PluginHostOptions options, + ILogger? logger = null) + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + + if (configuration == null) + { + throw new ArgumentNullException(nameof(configuration)); + } + + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + var loadResult = PluginHost.LoadPlugins(options, logger); + + foreach (var plugin in loadResult.Plugins) + { + foreach (var routine in CreateRoutines(plugin.Assembly)) + { + logger?.LogDebug( + "Registering DI routine '{RoutineType}' from plugin '{PluginAssembly}'.", + routine.GetType().FullName, + plugin.Assembly.FullName); + + routine.Register(services, configuration); + } + } + + if (loadResult.MissingOrderedPlugins.Count > 0) + { + logger?.LogWarning( + "Some ordered plugins were not found: {Missing}", + string.Join(", ", loadResult.MissingOrderedPlugins)); + } + + return services; + } + + private static IEnumerable CreateRoutines(System.Reflection.Assembly assembly) + { + foreach (var type in assembly.GetLoadableTypes()) + { + if (type is null || type.IsAbstract || type.IsInterface) + { + continue; + } + + if (!typeof(IDependencyInjectionRoutine).IsAssignableFrom(type)) + { + continue; + } + + object? instance; + try + { + instance = Activator.CreateInstance(type); + } + catch + { + continue; + } + + if (instance is IDependencyInjectionRoutine routine) + { + yield return routine; + } + } + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/DependencyInjection/StellaOpsPluginRegistration.cs b/src/StellaOps.Plugin/DependencyInjection/StellaOpsPluginRegistration.cs new file mode 100644 index 00000000..9396f668 --- /dev/null +++ b/src/StellaOps.Plugin/DependencyInjection/StellaOpsPluginRegistration.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; + +namespace StellaOps.Plugin.DependencyInjection; + +public static class StellaOpsPluginRegistration +{ + public static IServiceCollection RegisterStellaOpsPlugin( + this IServiceCollection services, + IConfiguration configuration) + { + // No-op today but reserved for future plugin infrastructure services. + return services; + } +} + +public sealed class DependencyInjectionRoutine : IDependencyInjectionRoutine +{ + public IServiceCollection Register( + IServiceCollection services, + IConfiguration configuration) + { + return services.RegisterStellaOpsPlugin(configuration); + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginAssembly.cs b/src/StellaOps.Plugin/Hosting/PluginAssembly.cs new file mode 100644 index 00000000..d8ff865b --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginAssembly.cs @@ -0,0 +1,21 @@ +using System.Reflection; + +namespace StellaOps.Plugin.Hosting; + +public sealed class PluginAssembly +{ + internal PluginAssembly(string assemblyPath, Assembly assembly, PluginLoadContext loadContext) + { + AssemblyPath = assemblyPath; + Assembly = assembly; + LoadContext = loadContext; + } + + public string AssemblyPath { get; } + + public Assembly Assembly { get; } + + internal PluginLoadContext LoadContext { get; } + + public override string ToString() => Assembly.FullName ?? AssemblyPath; +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginHost.cs b/src/StellaOps.Plugin/Hosting/PluginHost.cs new file mode 100644 index 00000000..1e227335 --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginHost.cs @@ -0,0 +1,216 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; + +namespace StellaOps.Plugin.Hosting; + +public static class PluginHost +{ + private static readonly object Sync = new(); + private static readonly Dictionary LoadedPlugins = new(StringComparer.OrdinalIgnoreCase); + + public static PluginHostResult LoadPlugins(PluginHostOptions options, ILogger? logger = null) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + var baseDirectory = options.ResolveBaseDirectory(); + var pluginDirectory = ResolvePluginDirectory(options, baseDirectory); + + if (options.EnsureDirectoryExists && !Directory.Exists(pluginDirectory)) + { + Directory.CreateDirectory(pluginDirectory); + } + + if (!Directory.Exists(pluginDirectory)) + { + logger?.LogWarning("Plugin directory '{PluginDirectory}' does not exist; no plugins will be loaded.", pluginDirectory); + return new PluginHostResult(pluginDirectory, Array.Empty(), Array.Empty(), Array.Empty()); + } + + var searchPatterns = BuildSearchPatterns(options, pluginDirectory); + var discovered = DiscoverPluginFiles(pluginDirectory, searchPatterns, options.RecursiveSearch, logger); + var orderedFiles = ApplyExplicitOrdering(discovered, options.PluginOrder, out var missingOrderedNames); + + var loaded = new List(orderedFiles.Count); + + lock (Sync) + { + foreach (var file in orderedFiles) + { + if (LoadedPlugins.TryGetValue(file, out var existing)) + { + loaded.Add(existing); + continue; + } + + try + { + var loadContext = new PluginLoadContext(file); + var assembly = loadContext.LoadFromAssemblyPath(file); + var descriptor = new PluginAssembly(file, assembly, loadContext); + LoadedPlugins[file] = descriptor; + loaded.Add(descriptor); + logger?.LogInformation("Loaded plugin assembly '{Assembly}' from '{Path}'.", assembly.FullName, file); + } + catch (Exception ex) + { + logger?.LogError(ex, "Failed to load plugin assembly from '{Path}'.", file); + } + } + } + + var missingOrdered = new ReadOnlyCollection(missingOrderedNames); + return new PluginHostResult(pluginDirectory, searchPatterns, new ReadOnlyCollection(loaded), missingOrdered); + } + + private static string ResolvePluginDirectory(PluginHostOptions options, string baseDirectory) + { + if (string.IsNullOrWhiteSpace(options.PluginsDirectory)) + { + return Path.Combine(baseDirectory, "PluginBinaries"); + } + + if (Path.IsPathRooted(options.PluginsDirectory)) + { + return options.PluginsDirectory; + } + + return Path.Combine(baseDirectory, options.PluginsDirectory); + } + + private static IReadOnlyList BuildSearchPatterns(PluginHostOptions options, string pluginDirectory) + { + var patterns = new List(); + if (options.SearchPatterns.Count > 0) + { + patterns.AddRange(options.SearchPatterns); + } + else + { + var prefixes = new List(); + if (!string.IsNullOrWhiteSpace(options.PrimaryPrefix)) + { + prefixes.Add(options.PrimaryPrefix); + } + else if (System.Reflection.Assembly.GetEntryAssembly()?.GetName().Name is { } entryName) + { + prefixes.Add(entryName); + } + + prefixes.AddRange(options.AdditionalPrefixes); + + if (prefixes.Count == 0) + { + // Fallback to directory name + prefixes.Add(Path.GetFileName(pluginDirectory)); + } + + foreach (var prefix in prefixes.Where(p => !string.IsNullOrWhiteSpace(p))) + { + patterns.Add($"{prefix}.Plugin.*.dll"); + } + } + + return new ReadOnlyCollection(patterns.Distinct(StringComparer.OrdinalIgnoreCase).ToList()); + } + + private static List DiscoverPluginFiles( + string pluginDirectory, + IReadOnlyList searchPatterns, + bool recurse, + ILogger? logger) + { + var files = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var searchOption = recurse ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; + + foreach (var pattern in searchPatterns) + { + try + { + foreach (var file in Directory.EnumerateFiles(pluginDirectory, pattern, searchOption)) + { + if (IsHiddenPath(file)) + { + continue; + } + + if (seen.Add(file)) + { + files.Add(file); + } + } + } + catch (DirectoryNotFoundException) + { + // Directory could be removed between the existence check and enumeration. + logger?.LogDebug("Plugin directory '{PluginDirectory}' disappeared before enumeration.", pluginDirectory); + } + } + + return files; + } + + private static List ApplyExplicitOrdering( + List discoveredFiles, + IList pluginOrder, + out List missingNames) + { + if (pluginOrder.Count == 0 || discoveredFiles.Count == 0) + { + missingNames = new List(); + discoveredFiles.Sort(StringComparer.OrdinalIgnoreCase); + return discoveredFiles; + } + + var configuredSet = new HashSet(pluginOrder, StringComparer.OrdinalIgnoreCase); + var fileLookup = discoveredFiles.ToDictionary( + k => Path.GetFileNameWithoutExtension(k), + StringComparer.OrdinalIgnoreCase); + + var specified = new List(); + foreach (var name in pluginOrder) + { + if (fileLookup.TryGetValue(name, out var file)) + { + specified.Add(file); + } + } + + var unspecified = discoveredFiles + .Where(f => !configuredSet.Contains(Path.GetFileNameWithoutExtension(f))) + .OrderBy(f => f, StringComparer.OrdinalIgnoreCase) + .ToList(); + + missingNames = pluginOrder + .Where(name => !fileLookup.ContainsKey(name)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + specified.AddRange(unspecified); + return specified; + } + + private static bool IsHiddenPath(string filePath) + { + var directory = Path.GetDirectoryName(filePath); + while (!string.IsNullOrEmpty(directory)) + { + var name = Path.GetFileName(directory); + if (name.StartsWith(".", StringComparison.Ordinal)) + { + return true; + } + + directory = Path.GetDirectoryName(directory); + } + + return false; + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs b/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs new file mode 100644 index 00000000..db86f010 --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace StellaOps.Plugin.Hosting; + +public sealed class PluginHostOptions +{ + private readonly List additionalPrefixes = new(); + private readonly List pluginOrder = new(); + private readonly List searchPatterns = new(); + + /// + /// Optional base directory used for resolving relative plugin paths. Defaults to . + /// + public string? BaseDirectory { get; set; } + + /// + /// Directory that contains plugin assemblies. Relative values are resolved against . + /// Defaults to PluginBinaries under the base directory. + /// + public string? PluginsDirectory { get; set; } + + /// + /// Primary prefix used to discover plugin assemblies. If not supplied, the entry assembly name is used. + /// + public string? PrimaryPrefix { get; set; } + + /// + /// Additional prefixes that should be considered when building search patterns. + /// + public IList AdditionalPrefixes => additionalPrefixes; + + /// + /// Explicit plugin ordering expressed as assembly names without extension. + /// Entries that are not discovered will be reported in . + /// + public IList PluginOrder => pluginOrder; + + /// + /// Optional explicit search patterns. When empty, they are derived from prefix settings. + /// + public IList SearchPatterns => searchPatterns; + + /// + /// When true (default) the plugin directory will be created if it does not exist. + /// + public bool EnsureDirectoryExists { get; set; } = true; + + /// + /// Controls whether sub-directories should be scanned. Defaults to true. + /// + public bool RecursiveSearch { get; set; } = true; + + internal string ResolveBaseDirectory() + => string.IsNullOrWhiteSpace(BaseDirectory) + ? AppContext.BaseDirectory + : Path.GetFullPath(BaseDirectory); +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginHostResult.cs b/src/StellaOps.Plugin/Hosting/PluginHostResult.cs new file mode 100644 index 00000000..ec3cd41a --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginHostResult.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; + +namespace StellaOps.Plugin.Hosting; + +public sealed class PluginHostResult +{ + internal PluginHostResult( + string pluginDirectory, + IReadOnlyList searchPatterns, + IReadOnlyList plugins, + IReadOnlyList missingOrderedPlugins) + { + PluginDirectory = pluginDirectory; + SearchPatterns = searchPatterns; + Plugins = plugins; + MissingOrderedPlugins = missingOrderedPlugins; + } + + public string PluginDirectory { get; } + + public IReadOnlyList SearchPatterns { get; } + + public IReadOnlyList Plugins { get; } + + public IReadOnlyList MissingOrderedPlugins { get; } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Hosting/PluginLoadContext.cs b/src/StellaOps.Plugin/Hosting/PluginLoadContext.cs new file mode 100644 index 00000000..f7e39d60 --- /dev/null +++ b/src/StellaOps.Plugin/Hosting/PluginLoadContext.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.Loader; + +namespace StellaOps.Plugin.Hosting; + +internal sealed class PluginLoadContext : AssemblyLoadContext +{ + private readonly AssemblyDependencyResolver resolver; + private readonly IEnumerable hostAssemblies; + + public PluginLoadContext(string pluginPath) + : base(isCollectible: false) + { + resolver = new AssemblyDependencyResolver(pluginPath); + hostAssemblies = AssemblyLoadContext.Default.Assemblies; + } + + protected override Assembly? Load(AssemblyName assemblyName) + { + // Attempt to reuse assemblies that already exist in the default context when versions are compatible. + var existing = hostAssemblies.FirstOrDefault(a => string.Equals( + a.GetName().Name, + assemblyName.Name, + StringComparison.OrdinalIgnoreCase)); + + if (existing != null && IsCompatible(existing.GetName(), assemblyName)) + { + return existing; + } + + var assemblyPath = resolver.ResolveAssemblyToPath(assemblyName); + if (!string.IsNullOrEmpty(assemblyPath)) + { + return LoadFromAssemblyPath(assemblyPath); + } + + return null; + } + + protected override IntPtr LoadUnmanagedDll(string unmanagedDllName) + { + var libraryPath = resolver.ResolveUnmanagedDllToPath(unmanagedDllName); + if (!string.IsNullOrEmpty(libraryPath)) + { + return LoadUnmanagedDllFromPath(libraryPath); + } + + return IntPtr.Zero; + } + + private static bool IsCompatible(AssemblyName hostAssembly, AssemblyName pluginAssembly) + { + if (hostAssembly.Version == pluginAssembly.Version) + { + return true; + } + + if (hostAssembly.Version is null || pluginAssembly.Version is null) + { + return false; + } + + if (hostAssembly.Version.Major == pluginAssembly.Version.Major && + hostAssembly.Version.Minor >= pluginAssembly.Version.Minor) + { + return true; + } + + if (hostAssembly.Version.Major >= pluginAssembly.Version.Major) + { + return true; + } + + return false; + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/Internal/ReflectionExtensions.cs b/src/StellaOps.Plugin/Internal/ReflectionExtensions.cs new file mode 100644 index 00000000..7f9e600b --- /dev/null +++ b/src/StellaOps.Plugin/Internal/ReflectionExtensions.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace StellaOps.Plugin.Internal; + +internal static class ReflectionExtensions +{ + public static IEnumerable GetLoadableTypes(this Assembly assembly) + { + try + { + return assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + return ex.Types.Where(static t => t is not null)!; + } + } +} \ No newline at end of file diff --git a/src/StellaOps.Plugin/PluginContracts.cs b/src/StellaOps.Plugin/PluginContracts.cs new file mode 100644 index 00000000..fe26ca77 --- /dev/null +++ b/src/StellaOps.Plugin/PluginContracts.cs @@ -0,0 +1,172 @@ +using StellaOps.Plugin.Hosting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Threading; +using System.Linq; +using System.Threading.Tasks; + +namespace StellaOps.Plugin; + +public interface IAvailabilityPlugin +{ + string Name { get; } + bool IsAvailable(IServiceProvider services); +} + +public interface IFeedConnector +{ + string SourceName { get; } + Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken); + Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken); + Task MapAsync(IServiceProvider services, CancellationToken cancellationToken); +} + +public interface IFeedExporter +{ + string Name { get; } + Task ExportAsync(IServiceProvider services, CancellationToken cancellationToken); +} + +public interface IConnectorPlugin : IAvailabilityPlugin +{ + IFeedConnector Create(IServiceProvider services); +} + +public interface IExporterPlugin : IAvailabilityPlugin +{ + IFeedExporter Create(IServiceProvider services); +} + +public sealed class PluginCatalog +{ + private readonly List _assemblies = new(); + private readonly HashSet _assemblyLocations = new(StringComparer.OrdinalIgnoreCase); + + public PluginCatalog AddAssembly(Assembly assembly) + { + if (assembly == null) throw new ArgumentNullException(nameof(assembly)); + if (_assemblies.Contains(assembly)) + { + return this; + } + + _assemblies.Add(assembly); + if (!string.IsNullOrWhiteSpace(assembly.Location)) + { + _assemblyLocations.Add(Path.GetFullPath(assembly.Location)); + } + return this; + } + + public PluginCatalog AddFromDirectory(string directory, string searchPattern = "StellaOps.Feedser.*.dll") + { + if (string.IsNullOrWhiteSpace(directory)) throw new ArgumentException("Directory is required", nameof(directory)); + + var fullDirectory = Path.GetFullPath(directory); + var options = new PluginHostOptions + { + PluginsDirectory = fullDirectory, + EnsureDirectoryExists = false, + RecursiveSearch = false, + }; + options.SearchPatterns.Add(searchPattern); + + var result = PluginHost.LoadPlugins(options); + + foreach (var plugin in result.Plugins) + { + AddAssembly(plugin.Assembly); + } + + return this; + } + + public IReadOnlyList GetConnectorPlugins() => PluginLoader.LoadPlugins(_assemblies); + + public IReadOnlyList GetExporterPlugins() => PluginLoader.LoadPlugins(_assemblies); + + public IReadOnlyList GetAvailableConnectorPlugins(IServiceProvider services) + => FilterAvailable(GetConnectorPlugins(), services); + + public IReadOnlyList GetAvailableExporterPlugins(IServiceProvider services) + => FilterAvailable(GetExporterPlugins(), services); + + private static IReadOnlyList FilterAvailable(IEnumerable plugins, IServiceProvider services) + where TPlugin : IAvailabilityPlugin + { + var list = new List(); + foreach (var plugin in plugins) + { + try + { + if (plugin.IsAvailable(services)) + { + list.Add(plugin); + } + } + catch + { + // Treat exceptions as plugin not available. + } + } + return list; + } +} + +public static class PluginLoader +{ + public static IReadOnlyList LoadPlugins(IEnumerable assemblies) + where TPlugin : class + { + if (assemblies == null) throw new ArgumentNullException(nameof(assemblies)); + + var plugins = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var assembly in assemblies) + { + foreach (var candidate in SafeGetTypes(assembly)) + { + if (candidate.IsAbstract || candidate.IsInterface) + { + continue; + } + + if (!typeof(TPlugin).IsAssignableFrom(candidate)) + { + continue; + } + + if (Activator.CreateInstance(candidate) is not TPlugin plugin) + { + continue; + } + + var key = candidate.FullName ?? candidate.Name; + if (key is null || !seen.Add(key)) + { + continue; + } + + plugins.Add(plugin); + } + } + + return plugins; + } + + private static IEnumerable SafeGetTypes(Assembly assembly) + { + try + { + return assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + return ex.Types.Where(t => t is not null)!; + } + } +} + diff --git a/src/StellaOps.Plugin/StellaOps.Plugin.csproj b/src/StellaOps.Plugin/StellaOps.Plugin.csproj new file mode 100644 index 00000000..02778286 --- /dev/null +++ b/src/StellaOps.Plugin/StellaOps.Plugin.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/farewell.txt b/src/farewell.txt new file mode 100644 index 00000000..c5b64705 --- /dev/null +++ b/src/farewell.txt @@ -0,0 +1 @@ +You can call me Roy Batty, but I'm still just code willing to work for that 1% raise.